From f1aa6149a31f07fbef649be2dfd6fc1931aca595 Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Tue, 3 May 2022 01:58:35 +0000 Subject: [PATCH 01/29] start balance transactions --- models/stripe__balance_transactions.sql | 89 +++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 models/stripe__balance_transactions.sql diff --git a/models/stripe__balance_transactions.sql b/models/stripe__balance_transactions.sql new file mode 100644 index 0000000..d6de421 --- /dev/null +++ b/models/stripe__balance_transactions.sql @@ -0,0 +1,89 @@ +with balance_transaction as ( + select + id as balance_transaction_id, + created as created_at, + fee, + net as net_change, + type, + status, + amount, + currency, + exchange_rate + from {{ var('balance_transaction') }} +), + +charges as ( + select + id as charges_id + customer as customer_id, + receipt_email, + payment_intent as payment_intent_id, + created as charges_created_at + from {{ var('charges')}} +), + +payment_intents as ( + select + * + from {{ var('payment_intents')}} +), + +cards as ( + select + * + from {{ var('card')}} +), + +payouts as ( + select + * + from {{ var('payouts')}} +), + +customers as ( + select + * + from {{ var('customers')}} +), + +refunds as ( + select + * + from {{ var('refunds')}} +), + +balance_transactions as ( + select + balance_transaction.*, + charges.*, + case (when balance_transaction.type = 'charges') then charges.amount end as customer_facing_amount, + case (when balance_transaction.type = 'charges') then charges.currency end as customer_facing_currency, + customers.description as customer_description + -- cards.brand as card_brand, + -- cards.funding as card_funding, + -- cards.country as card_country, + -- payouts.payouts_id, + -- payouts.arrival_date as payouts_expected_arrival_date, + -- payouts.status as payouts_status, + -- payouts.type as payouts_type, + -- payouts.description as payouts_description, + -- refunds.reason as refunds_reason + from balance_transaction + + -- left join charges + -- on charges.balance_transaction_id = balance_transaction.balance_transaction_id + -- left join customers + -- on charges.customer_id = customers.customer_id + -- left join payment_intents + -- on charges.payment_intents_id = payment_intents.payment_intents_id + -- left join cards + -- on charges.card_id = cards.card_id + -- left join payouts + -- on payouts.balance_transaction_id = balance_transaction.balance_transaction_id + -- left join refunds + -- on refunds.balance_transaction_id = balance_transaction.balance_transaction_id + -- left join charges as refunds_charges + -- on refunds.charges_id = refunds_charges.charges_id +) + +select * from balance_transactions \ No newline at end of file From 66118eb7749c33ef5740ee20fa10ccef32ef67ab Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Tue, 3 May 2022 23:25:42 +0000 Subject: [PATCH 02/29] balance transactions table --- dbt_project.yml | 12 +- models/schema.yml | 75 +++++++++++- models/stripe__balance_transactions.sql | 109 ++++++++---------- .../tmp/stg_stripe__balance_transactions.sql | 11 ++ models/tmp/stg_stripe__charges.sql | 11 ++ models/tmp/stg_stripe__charges_card.sql | 6 + models/tmp/stg_stripe__customers.sql | 4 + models/tmp/stg_stripe__payouts.sql | 8 ++ models/tmp/stg_stripe__refunds.sql | 8 ++ 9 files changed, 177 insertions(+), 67 deletions(-) create mode 100644 models/tmp/stg_stripe__balance_transactions.sql create mode 100644 models/tmp/stg_stripe__charges.sql create mode 100644 models/tmp/stg_stripe__charges_card.sql create mode 100644 models/tmp/stg_stripe__customers.sql create mode 100644 models/tmp/stg_stripe__payouts.sql create mode 100644 models/tmp/stg_stripe__refunds.sql diff --git a/dbt_project.yml b/dbt_project.yml index b9a8f65..a678fd4 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,4 +1,4 @@ -name: 'stripe' +name: 'airbyte_stripe' version: '0.1.0' require-dbt-version: ['>=1.0.0'] config-version: 2 @@ -18,9 +18,15 @@ clean-targets: models: airbyte_stripe: materialized: table - intermediate: - materialized: ephemeral + tmp: + materialized: view vars: airbyte_stripe: customers: "{{ source('airbyte_stripe','customers') }}" + balance_transactions: "{{ source('airbyte_stripe','balance_transactions') }}" + charges_card: "{{ source('airbyte_stripe','charges_card') }}" + charges: "{{ source('airbyte_stripe','charges') }}" + payment_intents: "{{ source('airbyte_stripe','payment_intents') }}" + payouts: "{{ source('airbyte_stripe','payouts') }}" + refunds: "{{ source('airbyte_stripe','refunds') }}" diff --git a/models/schema.yml b/models/schema.yml index 18762d1..3bcd64c 100644 --- a/models/schema.yml +++ b/models/schema.yml @@ -6,7 +6,12 @@ sources: database: "{% if target.type != 'spark'%}{{ var('stripe_database', target.database) }}{% endif %}" tables: - name: customers - + - name: balance_transactions + - name: charges_card + - name: charges + - name: payment_intents + - name: payouts + - name: refunds models: - name: stripe__customers description: "A table storing all customer info" @@ -16,3 +21,71 @@ models: tests: - unique - not_null + - name: stripe__balance_transactions + description: "A table storing all balance transactions and associated charge data" + columns: + - name: balance_transaction_id + description: "The unique identifier for the balance transaction" + tests: + - unique + - not_null + - name: created_at + description: "The date the balance transaction was created" + tests: + - not_null + - name: fee + description: "The fee associated with the balance transaction" + tests: + - not_null + - name: net_balance_change + description: "The net balance change due to the balance transaction" + tests: + - not_null + - name: type + description: "The type of balance transaction" + tests: + - not_null + - name: status + description: "The status of balance transaction" + tests: + - not_null + - name: amount + description: "The total transaction amount" + tests: + - not_null + - name: currency + description: "The currency associated with the balance transaction" + tests: + - not_null + - name: exchange_rate + description: "The exchange rate used for the balance transaction" + - name: charge_id + description: "The id of the charge associated with the balance transaction" + tests: + - not_null + - name: charge_created_at + description: "The date the charge was created" + - name: card_brand + description: "The card brand associated with the charge" + - name: card_funding + description: "The card funding type associated with the charge" + - name: card_country + description: "The country of the card charged" + - name: charge_amount + description: "The total charge amount" + - name: charge_currency + description: "The currency associated with the charge" + - name: customer_description + description: "The customer description associated with balance transaction" + - name: payout_id + description: "The id of the payout associated with the balance transaction" + - name: payout_arrival_date + description: "The date the payout is due to arrive" + - name: payout_status + description: "The status of payout" + - name: payout_type + description: "The type of payout" + - name: payout_description + description: "The user specified source of the payout" + - name: refund_reason + description: "The refund reason (if applicable)" \ No newline at end of file diff --git a/models/stripe__balance_transactions.sql b/models/stripe__balance_transactions.sql index d6de421..c518f55 100644 --- a/models/stripe__balance_transactions.sql +++ b/models/stripe__balance_transactions.sql @@ -1,89 +1,72 @@ -with balance_transaction as ( +with balance_transactions as ( select - id as balance_transaction_id, - created as created_at, - fee, - net as net_change, - type, - status, - amount, - currency, - exchange_rate - from {{ var('balance_transaction') }} + * + from {{ ref('stg_stripe__balance_transactions') }} ), -charges as ( +cards as ( select - id as charges_id - customer as customer_id, - receipt_email, - payment_intent as payment_intent_id, - created as charges_created_at - from {{ var('charges')}} + * + from {{ ref('stg_stripe__charges_card')}} ), -payment_intents as ( +charges as ( select - * - from {{ var('payment_intents')}} + * + from {{ ref('stg_stripe__charges')}} ), -cards as ( - select - * - from {{ var('card')}} -), - payouts as ( select * - from {{ var('payouts')}} + from {{ ref('stg_stripe__payouts')}} ), customers as ( select * - from {{ var('customers')}} + from {{ ref('stg_stripe__customers')}} ), refunds as ( select * - from {{ var('refunds')}} + from {{ ref('stg_stripe__refunds')}} ), -balance_transactions as ( - select - balance_transaction.*, - charges.*, - case (when balance_transaction.type = 'charges') then charges.amount end as customer_facing_amount, - case (when balance_transaction.type = 'charges') then charges.currency end as customer_facing_currency, - customers.description as customer_description - -- cards.brand as card_brand, - -- cards.funding as card_funding, - -- cards.country as card_country, - -- payouts.payouts_id, - -- payouts.arrival_date as payouts_expected_arrival_date, - -- payouts.status as payouts_status, - -- payouts.type as payouts_type, - -- payouts.description as payouts_description, - -- refunds.reason as refunds_reason - from balance_transaction - - -- left join charges - -- on charges.balance_transaction_id = balance_transaction.balance_transaction_id - -- left join customers - -- on charges.customer_id = customers.customer_id - -- left join payment_intents - -- on charges.payment_intents_id = payment_intents.payment_intents_id - -- left join cards - -- on charges.card_id = cards.card_id - -- left join payouts - -- on payouts.balance_transaction_id = balance_transaction.balance_transaction_id - -- left join refunds - -- on refunds.balance_transaction_id = balance_transaction.balance_transaction_id - -- left join charges as refunds_charges - -- on refunds.charges_id = refunds_charges.charges_id +balance_transactions_summary as ( + select + -- Balance Transactions + balance_transactions.*, + + -- Charges + charges.charge_id, + charges.created_at as charge_created_at, + cards.card_brand, + cards.card_funding, + cards.card_country, + case when balance_transactions.type = 'charges' then charges.amount end as charge_amount, + case when balance_transactions.type = 'charges' then charges.currency end as charge_currency, + + -- Customer details + customers.description as customer_description, + payouts.payout_id, + payouts.arrival_date as payout_arrival_date, + payouts.status as payout_status, + payouts.type as payout_type, + payouts.description as payout_description, + refunds.refund_reason + from balance_transactions + left join charges + using(balance_transaction_id) + left join cards + using(charges_hashid) + left join customers + using(customer_id) + left join payouts + using(balance_transaction_id) + left join refunds + using(balance_transaction_id) ) -select * from balance_transactions \ No newline at end of file +select * from balance_transactions_summary \ No newline at end of file diff --git a/models/tmp/stg_stripe__balance_transactions.sql b/models/tmp/stg_stripe__balance_transactions.sql new file mode 100644 index 0000000..939295b --- /dev/null +++ b/models/tmp/stg_stripe__balance_transactions.sql @@ -0,0 +1,11 @@ +select + id as balance_transaction_id, + created as created_at, + fee, + net as net_balance_change, + type, + status, + amount, + currency, + exchange_rate +from {{ var('balance_transactions') }} \ No newline at end of file diff --git a/models/tmp/stg_stripe__charges.sql b/models/tmp/stg_stripe__charges.sql new file mode 100644 index 0000000..910a639 --- /dev/null +++ b/models/tmp/stg_stripe__charges.sql @@ -0,0 +1,11 @@ +select + _airbyte_charges_hashid as charges_hashid, + id as charge_id, + customer as customer_id, + receipt_email, + payment_intent as payment_intent_id, + created as created_at, + amount, + currency, + balance_transaction as balance_transaction_id +from {{ var('charges')}} \ No newline at end of file diff --git a/models/tmp/stg_stripe__charges_card.sql b/models/tmp/stg_stripe__charges_card.sql new file mode 100644 index 0000000..a0434cf --- /dev/null +++ b/models/tmp/stg_stripe__charges_card.sql @@ -0,0 +1,6 @@ +select + _airbyte_charges_hashid as charges_hashid, + brand as card_brand, + funding as card_funding, + country as card_country +from {{ var('charges_card')}} \ No newline at end of file diff --git a/models/tmp/stg_stripe__customers.sql b/models/tmp/stg_stripe__customers.sql new file mode 100644 index 0000000..aa8e721 --- /dev/null +++ b/models/tmp/stg_stripe__customers.sql @@ -0,0 +1,4 @@ +select + id as customer_id, + description +from {{ var('customers')}} \ No newline at end of file diff --git a/models/tmp/stg_stripe__payouts.sql b/models/tmp/stg_stripe__payouts.sql new file mode 100644 index 0000000..f275f6d --- /dev/null +++ b/models/tmp/stg_stripe__payouts.sql @@ -0,0 +1,8 @@ +select + id as payout_id, + arrival_date, + status, + type, + description, + balance_transaction as balance_transaction_id +from {{ var('payouts')}} \ No newline at end of file diff --git a/models/tmp/stg_stripe__refunds.sql b/models/tmp/stg_stripe__refunds.sql new file mode 100644 index 0000000..ca0d262 --- /dev/null +++ b/models/tmp/stg_stripe__refunds.sql @@ -0,0 +1,8 @@ +select + id as refund_id, + charge as charge_id, + amount as refund_amount, + reason as refund_reason, + status as refund_status, + balance_transaction as balance_transaction_id +from {{ var('refunds')}} \ No newline at end of file From b9a355e0582ea4a9eaf9b92192d231893949e0c9 Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Wed, 4 May 2022 21:44:26 +0000 Subject: [PATCH 03/29] invoice_line_items table --- dbt_project.yml | 6 + models/schema.yml | 109 +++++++++++++++++- models/stripe__balance_transactions.sql | 6 +- models/stripe__invoice_line_items.sql | 100 ++++++++++++++++ models/tmp/stg_stripe__charges.sql | 5 +- models/tmp/stg_stripe__customers.sql | 3 +- models/tmp/stg_stripe__invoice_line_items.sql | 28 +++++ models/tmp/stg_stripe__invoices.sql | 18 +++ models/tmp/stg_stripe__plans.sql | 9 ++ models/tmp/stg_stripe__subscriptions.sql | 6 + 10 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 models/stripe__invoice_line_items.sql create mode 100644 models/tmp/stg_stripe__invoice_line_items.sql create mode 100644 models/tmp/stg_stripe__invoices.sql create mode 100644 models/tmp/stg_stripe__plans.sql create mode 100644 models/tmp/stg_stripe__subscriptions.sql diff --git a/dbt_project.yml b/dbt_project.yml index a678fd4..cec1c71 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -30,3 +30,9 @@ vars: payment_intents: "{{ source('airbyte_stripe','payment_intents') }}" payouts: "{{ source('airbyte_stripe','payouts') }}" refunds: "{{ source('airbyte_stripe','refunds') }}" + invoice_line_items: "{{ source('airbyte_stripe','invoice_line_items') }}" + invoice_line_items_period: "{{ source('airbyte_stripe','invoice_line_items_period') }}" + invoice_line_items_plan: "{{ source('airbyte_stripe','invoice_line_items_plan') }}" + invoices: "{{ source('airbyte_stripe','invoices') }}" + subscriptions: "{{ source('airbyte_stripe','subscriptions') }}" + plans: "{{ source('airbyte_stripe','plans') }}" diff --git a/models/schema.yml b/models/schema.yml index 3bcd64c..b90c830 100644 --- a/models/schema.yml +++ b/models/schema.yml @@ -12,6 +12,12 @@ sources: - name: payment_intents - name: payouts - name: refunds + - name: invoice_line_items + - name: invoice_line_items_period + - name: invoice_line_items_plan + - name: invoices + - name: subscriptions + - name: plans models: - name: stripe__customers description: "A table storing all customer info" @@ -88,4 +94,105 @@ models: - name: payout_description description: "The user specified source of the payout" - name: refund_reason - description: "The refund reason (if applicable)" \ No newline at end of file + description: "The refund reason (if applicable)" + - name: stripe__invoice_line_items + description: "A table storing all line items associated with an invoice" + columns: + - name: invoice_id + description: "The unique identifier for the balance transaction" + tests: + - unique + - not_null + - name: invoice_number + description: "The invoice number" + tests: + - not_null + - name: invoice_created_at + description: "The date the invoice was created" + tests: + - not_null + - name: status + description: "The status of the invoice" + tests: + - not_null + - name: due_date + description: "The due date of the invoice" + tests: + - not_null + - name: amount_due + description: "The total amount due for the invoice" + tests: + - not_null + - name: subtotal + description: "The subtotal amount for the invoice" + tests: + - not_null + - name: tax + description: "The tax amount for the invoice" + tests: + - not_null + - name: total + description: "The total amount for the invoice" + tests: + - not_null + - name: amount_paid + description: "The total amount paid for the invoice" + - name: amount_remaining + description: "The amount remaining to be paid for the invoice" + - name: attempt_count + description: "The number of attempts sent for this invoice" + - name: description + description: "The description of the invoice" + - name: invoice_line_item_id + description: "The unique id of the invoice line item" + tests: + - not_null + - unique + - name: line_item_description + description: "The description of the line item" + - name: line_item_amount + description: "The price of line item" + - name: line_item_quantity + description: "The total quantity of the line item" + - name: period_start + description: "The datetime of the period start for the invoice" + - name: period_end + description: "The datetime of the period end for the invoice" + - name: balance_transaction_id + description: "The unique id of the associated balance transaction" + tests: + - not_null + - name: charge_amount + description: "The amount charged for the line item" + - name: charge_status + description: "The status of the charge for the line item" + - name: charge_created_at + description: "The datetime when the charge for the line item was created" + - name: customer_description + description: "The description of the customer associated with line item purchase" + - name: customer_email + description: "The email of the customer associated with line item purchase" + - name: customer_id + description: "The unique id of the customer associated with line item purchase" + - name: subscription_id + description: "The unique id of the subscription purchase (if applicable)" + - name: subscription_billing + description: "The billing used for the subscription purchase" + - name: subscription_start_date + description: "The start date of the subscription purchase" + - name: subscription_end_date + description: "The end date of the subscription purchase" + - name: plan_id + description: "The plan associated with the purchased subscription" + - name: plan_is_active + description: "A flag for whether the subscription plan is active" + - name: plan_amount + description: "The total amount for the subscription plan" + - name: plan_interval + description: "The interval the subscription plan is charged at" + - name: plan_interval_count + description: "The number of intervals the subscription plan is charged" + - name: plan_nickname + description: "The nickname of the subscription plan" + - name: plan_product_id + description: "The unique id of the product associated with subscription plan" diff --git a/models/stripe__balance_transactions.sql b/models/stripe__balance_transactions.sql index c518f55..80c97ec 100644 --- a/models/stripe__balance_transactions.sql +++ b/models/stripe__balance_transactions.sql @@ -45,11 +45,11 @@ balance_transactions_summary as ( cards.card_brand, cards.card_funding, cards.card_country, - case when balance_transactions.type = 'charges' then charges.amount end as charge_amount, - case when balance_transactions.type = 'charges' then charges.currency end as charge_currency, + case when balance_transactions.type = 'charges' then charges.charge_amount end as charge_amount, + case when balance_transactions.type = 'charges' then charges.charge_currency end as charge_currency, -- Customer details - customers.description as customer_description, + customers.customer_description, payouts.payout_id, payouts.arrival_date as payout_arrival_date, payouts.status as payout_status, diff --git a/models/stripe__invoice_line_items.sql b/models/stripe__invoice_line_items.sql new file mode 100644 index 0000000..5173f01 --- /dev/null +++ b/models/stripe__invoice_line_items.sql @@ -0,0 +1,100 @@ +with invoices as ( + select + * + from {{ ref('stg_stripe__invoices') }} +), + +charges as ( + select + * + from {{ ref('stg_stripe__charges') }} +), + +invoice_line_items as ( + select + * + from {{ ref('stg_stripe__invoice_line_items') }} +), + +customers as ( + select + * + from {{ ref('stg_stripe__customers') }} + +), + +subscriptions as ( + select + * + from {{ ref('stg_stripe__subscriptions') }} + +), + +plans as ( + select + * + from {{ ref('stg_stripe__plans') }} +), + +line_items_summary as ( + select + -- Invoices + invoices.invoice_id, + invoices.invoice_number, + invoices.created_at as invoice_created_at, + invoices.status, + invoices.due_date, + invoices.amount_due, + invoices.subtotal, + invoices.tax, + invoices.total, + invoices.amount_paid, + invoices.amount_remaining, + invoices.attempt_count, + invoices.description, + + -- Line Items + invoice_line_items.invoice_line_item_id, + invoice_line_items.line_item_description, + invoice_line_items.line_item_amount, + invoice_line_items.line_item_quantity, + invoice_line_items.period_start, + invoice_line_items.period_end, + + -- Charges + charges.balance_transaction_id, + charges.charge_amount, + charges.charge_status, + charges.created_at as charge_created_at, + + -- Customer Details + customers.customer_description, + customers.customer_email, + customers.customer_id, + + -- Subscription Details + subscriptions.subscription_id, + subscriptions.subscription_billing, + subscriptions.start_date as subscription_start_date, + subscriptions.ended_at as subscription_ended_at, + plans.plan_id, + plans.plan_is_active, + plans.plan_amount, + plans.plan_interval, + plans.plan_interval_count, + plans.plan_nickname, + plans.plan_product_id + from invoices + left join charges + on charges.charge_id = invoices.charge_id + left join invoice_line_items + on invoices.invoice_id = invoice_line_items.invoice_id + left join subscriptions + on invoice_line_items.subscription_id = subscriptions.subscription_id + left join plans + on invoice_line_items.plan_id = plans.plan_id + left join customers + on invoices.customer_id = customers.customer_id +) + +select * from line_items_summary diff --git a/models/tmp/stg_stripe__charges.sql b/models/tmp/stg_stripe__charges.sql index 910a639..8ac1d06 100644 --- a/models/tmp/stg_stripe__charges.sql +++ b/models/tmp/stg_stripe__charges.sql @@ -5,7 +5,8 @@ select receipt_email, payment_intent as payment_intent_id, created as created_at, - amount, - currency, + status as charge_status, + amount as charge_amount, + currency as charge_currency, balance_transaction as balance_transaction_id from {{ var('charges')}} \ No newline at end of file diff --git a/models/tmp/stg_stripe__customers.sql b/models/tmp/stg_stripe__customers.sql index aa8e721..01167a5 100644 --- a/models/tmp/stg_stripe__customers.sql +++ b/models/tmp/stg_stripe__customers.sql @@ -1,4 +1,5 @@ select id as customer_id, - description + description as customer_description, + email as customer_email from {{ var('customers')}} \ No newline at end of file diff --git a/models/tmp/stg_stripe__invoice_line_items.sql b/models/tmp/stg_stripe__invoice_line_items.sql new file mode 100644 index 0000000..1766800 --- /dev/null +++ b/models/tmp/stg_stripe__invoice_line_items.sql @@ -0,0 +1,28 @@ +with invoice_line_items_period as ( + select + _airbyte_invoice_line_items_hashid, + start, + "end" + from {{ var('invoice_line_items_period') }} +), + +invoice_line_items_plan as ( + select + _airbyte_invoice_line_items_hashid, + id as plan_id + from {{ var('invoice_line_items_plan') }} +) + +select + id as invoice_line_item_id, + invoice as invoice_id, + subscription as subscription_id, + plan_id, + description as line_item_description, + amount as line_item_amount, + quantity as line_item_quantity, + start as period_start, + "end" as period_end +from {{ var('invoice_line_items') }} +left join invoice_line_items_period using(_airbyte_invoice_line_items_hashid) +left join invoice_line_items_plan using(_airbyte_invoice_line_items_hashid) \ No newline at end of file diff --git a/models/tmp/stg_stripe__invoices.sql b/models/tmp/stg_stripe__invoices.sql new file mode 100644 index 0000000..d135651 --- /dev/null +++ b/models/tmp/stg_stripe__invoices.sql @@ -0,0 +1,18 @@ +select + id as invoice_id, + created as created_at, + number as invoice_number, + description, + paid, + total, + subtotal, + tax, + amount_due, + amount_paid, + amount_remaining, + due_date, + attempt_count, + charge as charge_id, + status, + customer as customer_id +from {{ var('invoices') }} \ No newline at end of file diff --git a/models/tmp/stg_stripe__plans.sql b/models/tmp/stg_stripe__plans.sql new file mode 100644 index 0000000..032e5d7 --- /dev/null +++ b/models/tmp/stg_stripe__plans.sql @@ -0,0 +1,9 @@ +select + id as plan_id, + active as plan_is_active, + amount as plan_amount, + interval as plan_interval, + interval_count as plan_interval_count, + nickname as plan_nickname, + product as plan_product_id +from {{ var('plans') }} \ No newline at end of file diff --git a/models/tmp/stg_stripe__subscriptions.sql b/models/tmp/stg_stripe__subscriptions.sql new file mode 100644 index 0000000..18a8f99 --- /dev/null +++ b/models/tmp/stg_stripe__subscriptions.sql @@ -0,0 +1,6 @@ +select + id as subscription_id, + billing as subscription_billing, + start as start_date, + ended_at +from {{ var('subscriptions') }} \ No newline at end of file From b990468f0b797081f2c614fded1ea7930ff46e8d Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Thu, 5 May 2022 02:10:24 +0000 Subject: [PATCH 04/29] simplify joins --- models/stripe__invoice_line_items.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/stripe__invoice_line_items.sql b/models/stripe__invoice_line_items.sql index 5173f01..1536aed 100644 --- a/models/stripe__invoice_line_items.sql +++ b/models/stripe__invoice_line_items.sql @@ -86,15 +86,15 @@ line_items_summary as ( plans.plan_product_id from invoices left join charges - on charges.charge_id = invoices.charge_id + using(charge_id) left join invoice_line_items - on invoices.invoice_id = invoice_line_items.invoice_id + using(invoice_id) left join subscriptions on invoice_line_items.subscription_id = subscriptions.subscription_id left join plans on invoice_line_items.plan_id = plans.plan_id left join customers - on invoices.customer_id = customers.customer_id + using(customer_id) ) select * from line_items_summary From 8b69677a5b82ecd964b3bb540f1dd86294dda7fe Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Wed, 4 May 2022 20:27:02 -0700 Subject: [PATCH 05/29] consolidate stg tables --- models/stripe__balance_transactions.sql | 14 +++---------- models/stripe__invoice_line_items.sql | 2 +- models/tmp/stg_stripe__charges.sql | 20 ++++++++++++++++--- models/tmp/stg_stripe__charges_card.sql | 6 ------ models/tmp/stg_stripe__invoice_line_items.sql | 8 +++++--- 5 files changed, 26 insertions(+), 24 deletions(-) delete mode 100644 models/tmp/stg_stripe__charges_card.sql diff --git a/models/stripe__balance_transactions.sql b/models/stripe__balance_transactions.sql index 80c97ec..ab341bb 100644 --- a/models/stripe__balance_transactions.sql +++ b/models/stripe__balance_transactions.sql @@ -4,12 +4,6 @@ with balance_transactions as ( from {{ ref('stg_stripe__balance_transactions') }} ), -cards as ( - select - * - from {{ ref('stg_stripe__charges_card')}} -), - charges as ( select * @@ -42,9 +36,9 @@ balance_transactions_summary as ( -- Charges charges.charge_id, charges.created_at as charge_created_at, - cards.card_brand, - cards.card_funding, - cards.card_country, + charges.card_brand, + charges.card_funding, + charges.card_country, case when balance_transactions.type = 'charges' then charges.charge_amount end as charge_amount, case when balance_transactions.type = 'charges' then charges.charge_currency end as charge_currency, @@ -59,8 +53,6 @@ balance_transactions_summary as ( from balance_transactions left join charges using(balance_transaction_id) - left join cards - using(charges_hashid) left join customers using(customer_id) left join payouts diff --git a/models/stripe__invoice_line_items.sql b/models/stripe__invoice_line_items.sql index 1536aed..988ccd6 100644 --- a/models/stripe__invoice_line_items.sql +++ b/models/stripe__invoice_line_items.sql @@ -94,7 +94,7 @@ line_items_summary as ( left join plans on invoice_line_items.plan_id = plans.plan_id left join customers - using(customer_id) + on customers.customer_id = invoices.customer_id ) select * from line_items_summary diff --git a/models/tmp/stg_stripe__charges.sql b/models/tmp/stg_stripe__charges.sql index 8ac1d06..7a170ea 100644 --- a/models/tmp/stg_stripe__charges.sql +++ b/models/tmp/stg_stripe__charges.sql @@ -1,5 +1,13 @@ +with charges_card as ( + select + _airbyte_charges_hashid, + brand as card_brand, + funding as card_funding, + country as card_country + from {{ var('charges_card')}} +) + select - _airbyte_charges_hashid as charges_hashid, id as charge_id, customer as customer_id, receipt_email, @@ -8,5 +16,11 @@ select status as charge_status, amount as charge_amount, currency as charge_currency, - balance_transaction as balance_transaction_id -from {{ var('charges')}} \ No newline at end of file + captured as charge_is_captured, + balance_transaction as balance_transaction_id, + card_brand, + card_funding, + card_country +from {{ var('charges')}} +left join charges_card + using(_airbyte_charges_hashid) \ No newline at end of file diff --git a/models/tmp/stg_stripe__charges_card.sql b/models/tmp/stg_stripe__charges_card.sql deleted file mode 100644 index a0434cf..0000000 --- a/models/tmp/stg_stripe__charges_card.sql +++ /dev/null @@ -1,6 +0,0 @@ -select - _airbyte_charges_hashid as charges_hashid, - brand as card_brand, - funding as card_funding, - country as card_country -from {{ var('charges_card')}} \ No newline at end of file diff --git a/models/tmp/stg_stripe__invoice_line_items.sql b/models/tmp/stg_stripe__invoice_line_items.sql index 1766800..2a0c5ae 100644 --- a/models/tmp/stg_stripe__invoice_line_items.sql +++ b/models/tmp/stg_stripe__invoice_line_items.sql @@ -9,7 +9,7 @@ with invoice_line_items_period as ( invoice_line_items_plan as ( select _airbyte_invoice_line_items_hashid, - id as plan_id + id as plan_id from {{ var('invoice_line_items_plan') }} ) @@ -24,5 +24,7 @@ select start as period_start, "end" as period_end from {{ var('invoice_line_items') }} -left join invoice_line_items_period using(_airbyte_invoice_line_items_hashid) -left join invoice_line_items_plan using(_airbyte_invoice_line_items_hashid) \ No newline at end of file +left join invoice_line_items_period + using(_airbyte_invoice_line_items_hashid) +left join invoice_line_items_plan + using(_airbyte_invoice_line_items_hashid) \ No newline at end of file From 647e7d5a533bf7fc746458d64428eb166615dcb4 Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Wed, 4 May 2022 20:56:43 -0700 Subject: [PATCH 06/29] add dbt_date --- dbt_project.yml | 2 ++ packages.yml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index cec1c71..7fc22f8 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -36,3 +36,5 @@ vars: invoices: "{{ source('airbyte_stripe','invoices') }}" subscriptions: "{{ source('airbyte_stripe','subscriptions') }}" plans: "{{ source('airbyte_stripe','plans') }}" + + "dbt_date:time_zone": "UTC" diff --git a/packages.yml b/packages.yml index 6482422..f695347 100644 --- a/packages.yml +++ b/packages.yml @@ -1,5 +1,5 @@ packages: - - package: dbt-labs/dbt_utils - version: 0.8.4 + - package: calogica/dbt_date + version: [">=0.5.0", "<0.6.0"] - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] \ No newline at end of file From cea9dd998f93ed545cfab67b21b15ebd096f805b Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Wed, 4 May 2022 20:57:04 -0700 Subject: [PATCH 07/29] use dbt_date to convert to datetime --- models/tmp/stg_stripe__balance_transactions.sql | 6 ++++-- models/tmp/stg_stripe__charges.sql | 2 +- models/tmp/stg_stripe__invoice_line_items.sql | 8 ++++---- models/tmp/stg_stripe__invoices.sql | 4 ++-- models/tmp/stg_stripe__payouts.sql | 2 +- models/tmp/stg_stripe__subscriptions.sql | 4 ++-- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/models/tmp/stg_stripe__balance_transactions.sql b/models/tmp/stg_stripe__balance_transactions.sql index 939295b..68beb52 100644 --- a/models/tmp/stg_stripe__balance_transactions.sql +++ b/models/tmp/stg_stripe__balance_transactions.sql @@ -1,11 +1,13 @@ select id as balance_transaction_id, - created as created_at, + {{ dbt_date.from_unixtimestamp('created') }} as created_at, + {{ dbt_date.from_unixtimestamp('available_on') }} as available_on, fee, net as net_balance_change, type, status, amount, currency, - exchange_rate + exchange_rate, + source from {{ var('balance_transactions') }} \ No newline at end of file diff --git a/models/tmp/stg_stripe__charges.sql b/models/tmp/stg_stripe__charges.sql index 7a170ea..55e438a 100644 --- a/models/tmp/stg_stripe__charges.sql +++ b/models/tmp/stg_stripe__charges.sql @@ -12,7 +12,7 @@ select customer as customer_id, receipt_email, payment_intent as payment_intent_id, - created as created_at, + {{ dbt_date.from_unixtimestamp('created') }} as created_at, status as charge_status, amount as charge_amount, currency as charge_currency, diff --git a/models/tmp/stg_stripe__invoice_line_items.sql b/models/tmp/stg_stripe__invoice_line_items.sql index 2a0c5ae..c4b98da 100644 --- a/models/tmp/stg_stripe__invoice_line_items.sql +++ b/models/tmp/stg_stripe__invoice_line_items.sql @@ -1,8 +1,8 @@ with invoice_line_items_period as ( select _airbyte_invoice_line_items_hashid, - start, - "end" + {{ dbt_date.from_unixtimestamp('start') }} as period_start, + {{ dbt_date.from_unixtimestamp('"end"') }} as period_end from {{ var('invoice_line_items_period') }} ), @@ -21,8 +21,8 @@ select description as line_item_description, amount as line_item_amount, quantity as line_item_quantity, - start as period_start, - "end" as period_end + period_start, + period_end from {{ var('invoice_line_items') }} left join invoice_line_items_period using(_airbyte_invoice_line_items_hashid) diff --git a/models/tmp/stg_stripe__invoices.sql b/models/tmp/stg_stripe__invoices.sql index d135651..14085a7 100644 --- a/models/tmp/stg_stripe__invoices.sql +++ b/models/tmp/stg_stripe__invoices.sql @@ -1,6 +1,6 @@ select id as invoice_id, - created as created_at, + {{ dbt_date.from_unixtimestamp('created') }} as created_at, number as invoice_number, description, paid, @@ -10,7 +10,7 @@ select amount_due, amount_paid, amount_remaining, - due_date, + {{ dbt_date.from_unixtimestamp('due_date') }} as due_date, attempt_count, charge as charge_id, status, diff --git a/models/tmp/stg_stripe__payouts.sql b/models/tmp/stg_stripe__payouts.sql index f275f6d..998e40c 100644 --- a/models/tmp/stg_stripe__payouts.sql +++ b/models/tmp/stg_stripe__payouts.sql @@ -1,6 +1,6 @@ select id as payout_id, - arrival_date, + {{ dbt_date.from_unixtimestamp('arrival_date') }} as arrival_date, status, type, description, diff --git a/models/tmp/stg_stripe__subscriptions.sql b/models/tmp/stg_stripe__subscriptions.sql index 18a8f99..3eb77f1 100644 --- a/models/tmp/stg_stripe__subscriptions.sql +++ b/models/tmp/stg_stripe__subscriptions.sql @@ -1,6 +1,6 @@ select id as subscription_id, billing as subscription_billing, - start as start_date, - ended_at + {{ dbt_date.from_unixtimestamp('start') }} as start_date, + {{ dbt_date.from_unixtimestamp('ended_at') }} as ended_at from {{ var('subscriptions') }} \ No newline at end of file From 499fd0186e7edb441a301b830607c0d3aa683b75 Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Wed, 4 May 2022 20:57:17 -0700 Subject: [PATCH 08/29] daily_transactions view --- models/tmp/int_stripe__daily_transactions.sql | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 models/tmp/int_stripe__daily_transactions.sql diff --git a/models/tmp/int_stripe__daily_transactions.sql b/models/tmp/int_stripe__daily_transactions.sql new file mode 100644 index 0000000..f03c845 --- /dev/null +++ b/models/tmp/int_stripe__daily_transactions.sql @@ -0,0 +1,95 @@ +with balance_transactions as ( + select + * + from {{ ref('stripe__balance_transactions') }} +), + +incomplete_charges as ( + select + created_at, + customer_id, + charge_amount + from {{ ref('stg_stripe__charges')}} + where not charge_is_captured +), + +daily_balance_transactions as ( + select + case + when type = 'payout' + then {{ dbt_utils.date_trunc("day", 'available_on') }} + else {{ dbt_utils.date_trunc("day", 'created_at') }} + end as date, + sum(case when type in ('charge', 'payment') + then amount + else 0 end) as total_sales, + sum(case when type in ('payment_refund', 'refund') + then amount + else 0 end) as total_refunds, + sum(case when type = 'adjustment' + then amount + else 0 end) as total_adjustments, + sum(case when type not in ('charge', 'payment', 'payment_refund', 'refund', 'adjustment', 'payout') and type not like '%transfer%' + then amount + else 0 end) as total_other_transactions, + sum(case when type <> 'payout' and type not like '%transfer%' + then amount + else 0 end) as total_gross_transaction_amount, + sum(case when type <> 'payout' and type not like '%transfer%' + then net_balance_change + else 0 end) as total_net_transactions, + sum(case when type = 'payout' or type like '%transfer%' + then fee * -1.0 + else 0 end) as total_payout_fees, + sum(case when type = 'payout' or type like '%transfer%' + then amount + else 0 end) as total_gross_payout_amount, + sum(case when type = 'payout' or type like '%transfer%' + then fee * -1.0 + else net_balance_change end) as daily_net_activity, + sum(case when type in ('payment', 'charge') + then 1 + else 0 end) as total_sales_count, + sum(case when type = 'payout' + then 1 + else 0 end) as total_payouts_count, + count(distinct case when type = 'adjustment' + then coalesce(source, payout_id) + else null end) as total_adjustments_count + from balance_transactions + {{ dbt_utils.group_by(1) }} +), + +daily_failed_charges as ( + select + {{ dbt_utils.date_trunc("day",'created_at') }} as date, + count(*) as total_failed_charge_count, + sum(charge_amount) as total_failed_charge_amount + from incomplete_charges + {{ dbt_utils.group_by(1) }} +), + +daily_transactions as ( + select + daily_balance_transactions.date, + daily_balance_transactions.total_sales/100.0 as total_sales, + daily_balance_transactions.total_refunds/100.0 as total_refunds, + daily_balance_transactions.total_adjustments/100.0 as total_adjustments, + daily_balance_transactions.total_other_transactions/100.0 as total_other_transactions, + daily_balance_transactions.total_gross_transaction_amount/100.0 as total_gross_transaction_amount, + daily_balance_transactions.total_net_transactions/100.0 as total_net_transactions, + daily_balance_transactions.total_payout_fees/100.0 as total_payout_fees, + daily_balance_transactions.total_gross_payout_amount/100.0 as total_gross_payout_amount, + daily_balance_transactions.daily_net_activity/100.0 as daily_net_activity, + (daily_balance_transactions.daily_net_activity + daily_balance_transactions.total_gross_payout_amount)/100.0 as daily_end_balance, + daily_balance_transactions.total_sales_count, + daily_balance_transactions.total_payouts_count, + daily_balance_transactions.total_adjustments_count, + coalesce(daily_failed_charges.total_failed_charge_count, 0) as total_failed_charge_count, + coalesce(daily_failed_charges.total_failed_charge_amount/100, 0) as total_failed_charge_amount + from daily_balance_transactions + left join daily_failed_charges + using(date) +) + +select * from daily_transactions \ No newline at end of file From be5db6fab03d26854b6fb8ca535ed5954ede0a6b Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Wed, 4 May 2022 21:06:10 -0700 Subject: [PATCH 09/29] fix indents --- models/tmp/int_stripe__daily_transactions.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/tmp/int_stripe__daily_transactions.sql b/models/tmp/int_stripe__daily_transactions.sql index f03c845..507c845 100644 --- a/models/tmp/int_stripe__daily_transactions.sql +++ b/models/tmp/int_stripe__daily_transactions.sql @@ -81,12 +81,12 @@ daily_transactions as ( daily_balance_transactions.total_payout_fees/100.0 as total_payout_fees, daily_balance_transactions.total_gross_payout_amount/100.0 as total_gross_payout_amount, daily_balance_transactions.daily_net_activity/100.0 as daily_net_activity, - (daily_balance_transactions.daily_net_activity + daily_balance_transactions.total_gross_payout_amount)/100.0 as daily_end_balance, + (daily_balance_transactions.daily_net_activity + daily_balance_transactions.total_gross_payout_amount)/100.0 as daily_end_balance, daily_balance_transactions.total_sales_count, daily_balance_transactions.total_payouts_count, daily_balance_transactions.total_adjustments_count, - coalesce(daily_failed_charges.total_failed_charge_count, 0) as total_failed_charge_count, - coalesce(daily_failed_charges.total_failed_charge_amount/100, 0) as total_failed_charge_amount + coalesce(daily_failed_charges.total_failed_charge_count, 0) as total_failed_charge_count, + coalesce(daily_failed_charges.total_failed_charge_amount/100, 0) as total_failed_charge_amount from daily_balance_transactions left join daily_failed_charges using(date) From b4224d8c5cff62cfad58a9ce162989966b2e2971 Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Wed, 4 May 2022 21:38:30 -0700 Subject: [PATCH 10/29] start customers table --- models/stripe__customers.sql | 210 +++++++++++++++++++++++++++++++---- 1 file changed, 187 insertions(+), 23 deletions(-) diff --git a/models/stripe__customers.sql b/models/stripe__customers.sql index 1f2360b..84c0afa 100644 --- a/models/stripe__customers.sql +++ b/models/stripe__customers.sql @@ -1,33 +1,197 @@ -with commit_comments_customer as ( +with balance_transactions as ( + select + * + from {{ ref('stripe__balance_transactions') }} +), + +incomplete_charges as ( + select + created_at, + customer_id, + charge_amount + from {{ ref('stg_stripe__charges')}} + where not charge_is_captured +), + +customers as ( select - id as customer_id, - url, - type, - login as customer_name - from {{ var('commit_comments_customer') }} + * + from {{ var('stg_stripe__customers') }} ), -commit_comment_reactions_customer as ( +transactions_by_customer as ( select - id as customer_id, - url, - type, - login as customer_name - from {{ var('commit_comment_reactions_customer') }} + customer_id, + sum(case when type in ('charge', 'payment') + then amount + else 0 + end) as total_sales, + sum(case when type in ('payment_refund', 'refund') + then amount + else 0 + end) as total_refunds, + sum(amount) as total_gross_transaction_amount, + sum(fee) as total_fees, + sum(net) as total_net_transaction_amount, + sum(case when type in ('charge', 'payment') + then 1 + else 0 + end) as total_sales_count, + sum(case when type in ('payment_refund', 'refund') + then 1 + else 0 + end) as total_refund_count, + sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + then amount + else 0 + end) as sales_this_month, + sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + then amount + else 0 + end) as refunds_this_month, + sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + then amount + else 0 + end) as gross_transaction_amount_this_month, + sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + then fee + else 0 + end) as fees_this_month, + sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + then net_balance_change + else 0 + end) as net_transaction_amount_this_month, + sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + then 1 + else 0 + end) as sales_count_this_month, + sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + then 1 + else 0 + end) as refund_count_this_month, + min(case when type in ('charge', 'payment') + then {{ date_timezone('created_at') }} + else null + end) as first_sale_date, + max(case when type in ('charge', 'payment') + then {{ date_timezone('created_at') }} + else null + end) as most_recent_sale_date + from balance_transactions + where type in ('payment', 'charge', 'payment_refund', 'refund') + {{ dbt_utils.group_by(1) }} ), -customers_union as ( - select * from commit_comments_customer - union all - select * from commit_comment_reactions_customer - union all - select * from issue_comment_reactions_customer +failed_charges_by_customer as ( + select + customer_id, + count(*) as total_failed_charge_count, + sum(amount) as total_failed_charge_amount, + sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + then 1 + else 0 + end) as failed_charge_count_this_month, + sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + then amount + else 0 + end) as failed_charge_amount_this_month + from incomplete_charges + {{ dbt_utils.group_by(1) }} ), -customers as ( - select * - from customers_union - group by 1,2 +no_customer_transactions_overview as ( + select + 'No Associated Customer' as customer_description, + customer.email, + customer.created_at as customer_created_at, + customer.is_delinquent, + coalesce(transactions_grouped.total_sales/100.0, 0) as total_sales, + coalesce(transactions_grouped.total_refunds/100.0, 0) as total_refunds, + coalesce(transactions_grouped.total_gross_transaction_amount/100.0, 0) as total_gross_transaction_amount, + coalesce(transactions_grouped.total_fees/100.0, 0) as total_fees, + coalesce(transactions_grouped.total_net_transaction_amount/100.0, 0) as total_net_transaction_amount, + coalesce(transactions_grouped.total_sales_count, 0) as total_sales_count, + coalesce(transactions_grouped.total_refund_count, 0) as total_refund_count, + coalesce(transactions_grouped.sales_this_month/100.0, 0) as sales_this_month, + coalesce(transactions_grouped.refunds_this_month/100.0, 0) as refunds_this_month, + coalesce(transactions_grouped.gross_transaction_amount_this_month/100.0, 0) as gross_transaction_amount_this_month, + coalesce(transactions_grouped.fees_this_month/100.0, 0) as fees_this_month, + coalesce(transactions_grouped.net_transaction_amount_this_month/100.0, 0) as net_transaction_amount_this_month, + coalesce(transactions_grouped.sales_count_this_month, 0) as sales_count_this_month, + coalesce(transactions_grouped.refund_count_this_month, 0) as refund_count_this_month, + transactions_grouped.first_sale_date, + transactions_grouped.most_recent_sale_date, + 0 as total_failed_charge_count, + 0 as total_failed_charge_amount, + 0 as failed_charge_count_this_month, + 0 as failed_charge_amount_this_month, + customer.currency as customer_currency, + customer.default_card_id, + customer.shipping_name, + customer.shipping_address_line_1, + customer.shipping_address_line_2, + customer.shipping_address_city, + customer.shipping_address_state, + customer.shipping_address_country, + customer.shipping_address_postal_code, + customer.shipping_phone + from transactions_by_customer + left join customers + using(customer_id) + where customer.customer_id is null and customer.description is null +), + +customer_transactions_overview as ( + select + coalesce(customer.description, customer.customer_id) as customer_description, + customer.email, + customer.created_at as customer_created_at, + customer.is_delinquent, + coalesce(transactions_grouped.total_sales/100.0, 0) as total_sales, + coalesce(transactions_grouped.total_refunds/100.0, 0) as total_refunds, + coalesce(transactions_grouped.total_gross_transaction_amount/100.0, 0) as total_gross_transaction_amount, + coalesce(transactions_grouped.total_fees/100.0, 0) as total_fees, + coalesce(transactions_grouped.total_net_transaction_amount/100.0, 0) as total_net_transaction_amount, + coalesce(transactions_grouped.total_sales_count, 0) as total_sales_count, + coalesce(transactions_grouped.total_refund_count, 0) as total_refund_count, + coalesce(transactions_grouped.sales_this_month/100.0, 0) as sales_this_month, + coalesce(transactions_grouped.refunds_this_month/100.0, 0) as refunds_this_month, + coalesce(transactions_grouped.gross_transaction_amount_this_month/100.0, 0) as gross_transaction_amount_this_month, + coalesce(transactions_grouped.fees_this_month/100.0, 0) as fees_this_month, + coalesce(transactions_grouped.net_transaction_amount_this_month/100.0, 0) as net_transaction_amount_this_month, + coalesce(transactions_grouped.sales_count_this_month, 0) as sales_count_this_month, + coalesce(transactions_grouped.refund_count_this_month, 0) as refund_count_this_month, + transactions_grouped.first_sale_date, + transactions_grouped.most_recent_sale_date, + coalesce(failed_charges_by_customer.total_failed_charge_count, 0) as total_failed_charge_count, + coalesce(failed_charges_by_customer.total_failed_charge_amount/100, 0) as total_failed_charge_amount, + coalesce(failed_charges_by_customer.failed_charge_count_this_month, 0) as failed_charge_count_this_month, + coalesce(failed_charges_by_customer.failed_charge_amount_this_month/100, 0) as failed_charge_amount_this_month, + customer.currency as customer_currency, + customer.default_card_id, + customer.shipping_name, + customer.shipping_address_line_1, + customer.shipping_address_line_2, + customer.shipping_address_city, + customer.shipping_address_state, + customer.shipping_address_country, + customer.shipping_address_postal_code, + customer.shipping_phone + from customer + left join transactions_by_customer + using(customer_id) + left join failed_charges_by_customer + using(customer_id) +), + +customer_overview as ( + select + * + from no_customer_transactions_overview + union all + select + * + from customer_transactions_overview ) -select * from customers +select * from customer_overview \ No newline at end of file From 6c43aa9197a6e926262d248e4415495ef6a2c56a Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Wed, 4 May 2022 22:13:55 -0700 Subject: [PATCH 11/29] v1 customers table --- dbt_project.yml | 1 + models/schema.yml | 1 + models/stripe__balance_transactions.sql | 3 +- models/stripe__customers.sql | 154 ++++++++++++------------ models/tmp/stg_stripe__customers.sql | 30 ++++- 5 files changed, 109 insertions(+), 80 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 7fc22f8..763d2bd 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -24,6 +24,7 @@ models: vars: airbyte_stripe: customers: "{{ source('airbyte_stripe','customers') }}" + customers_address: "{{ source('airbyte_stripe','customers_address') }}" balance_transactions: "{{ source('airbyte_stripe','balance_transactions') }}" charges_card: "{{ source('airbyte_stripe','charges_card') }}" charges: "{{ source('airbyte_stripe','charges') }}" diff --git a/models/schema.yml b/models/schema.yml index b90c830..02a90c1 100644 --- a/models/schema.yml +++ b/models/schema.yml @@ -6,6 +6,7 @@ sources: database: "{% if target.type != 'spark'%}{{ var('stripe_database', target.database) }}{% endif %}" tables: - name: customers + - name: customers_address - name: balance_transactions - name: charges_card - name: charges diff --git a/models/stripe__balance_transactions.sql b/models/stripe__balance_transactions.sql index ab341bb..c6f9375 100644 --- a/models/stripe__balance_transactions.sql +++ b/models/stripe__balance_transactions.sql @@ -42,7 +42,8 @@ balance_transactions_summary as ( case when balance_transactions.type = 'charges' then charges.charge_amount end as charge_amount, case when balance_transactions.type = 'charges' then charges.charge_currency end as charge_currency, - -- Customer details + -- Customer details + charges.customer_id, customers.customer_description, payouts.payout_id, payouts.arrival_date as payout_arrival_date, diff --git a/models/stripe__customers.sql b/models/stripe__customers.sql index 84c0afa..45abff2 100644 --- a/models/stripe__customers.sql +++ b/models/stripe__customers.sql @@ -5,7 +5,7 @@ with balance_transactions as ( ), incomplete_charges as ( - select + select created_at, customer_id, charge_amount @@ -16,7 +16,7 @@ incomplete_charges as ( customers as ( select * - from {{ var('stg_stripe__customers') }} + from {{ ref('stg_stripe__customers') }} ), transactions_by_customer as ( @@ -32,7 +32,7 @@ transactions_by_customer as ( end) as total_refunds, sum(amount) as total_gross_transaction_amount, sum(fee) as total_fees, - sum(net) as total_net_transaction_amount, + sum(net_balance_change) as total_net_transaction_amount, sum(case when type in ('charge', 'payment') then 1 else 0 @@ -41,40 +41,40 @@ transactions_by_customer as ( then 1 else 0 end) as total_refund_count, - sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then amount else 0 end) as sales_this_month, - sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then amount else 0 end) as refunds_this_month, - sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + sum(case when {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then amount else 0 end) as gross_transaction_amount_this_month, - sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + sum(case when {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then fee else 0 end) as fees_this_month, - sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + sum(case when {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then net_balance_change else 0 end) as net_transaction_amount_this_month, - sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then 1 else 0 end) as sales_count_this_month, - sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then 1 else 0 end) as refund_count_this_month, min(case when type in ('charge', 'payment') - then {{ date_timezone('created_at') }} + then {{ dbt_utils.date_trunc("day", 'created_at') }} else null end) as first_sale_date, max(case when type in ('charge', 'payment') - then {{ date_timezone('created_at') }} + then {{ dbt_utils.date_trunc("day", 'created_at') }} else null end) as most_recent_sale_date from balance_transactions @@ -86,13 +86,13 @@ failed_charges_by_customer as ( select customer_id, count(*) as total_failed_charge_count, - sum(amount) as total_failed_charge_amount, - sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} + sum(charge_amount) as total_failed_charge_amount, + sum(case when {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then 1 else 0 end) as failed_charge_count_this_month, - sum(case when {{ dbt_utils.date_trunc('month', date_timezone('created_at')) }} = {{ dbt_utils.date_trunc('month', date_timezone(dbt_utils.current_timestamp())) }} - then amount + sum(case when {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} + then charge_amount else 0 end) as failed_charge_amount_this_month from incomplete_charges @@ -102,82 +102,82 @@ failed_charges_by_customer as ( no_customer_transactions_overview as ( select 'No Associated Customer' as customer_description, - customer.email, - customer.created_at as customer_created_at, - customer.is_delinquent, - coalesce(transactions_grouped.total_sales/100.0, 0) as total_sales, - coalesce(transactions_grouped.total_refunds/100.0, 0) as total_refunds, - coalesce(transactions_grouped.total_gross_transaction_amount/100.0, 0) as total_gross_transaction_amount, - coalesce(transactions_grouped.total_fees/100.0, 0) as total_fees, - coalesce(transactions_grouped.total_net_transaction_amount/100.0, 0) as total_net_transaction_amount, - coalesce(transactions_grouped.total_sales_count, 0) as total_sales_count, - coalesce(transactions_grouped.total_refund_count, 0) as total_refund_count, - coalesce(transactions_grouped.sales_this_month/100.0, 0) as sales_this_month, - coalesce(transactions_grouped.refunds_this_month/100.0, 0) as refunds_this_month, - coalesce(transactions_grouped.gross_transaction_amount_this_month/100.0, 0) as gross_transaction_amount_this_month, - coalesce(transactions_grouped.fees_this_month/100.0, 0) as fees_this_month, - coalesce(transactions_grouped.net_transaction_amount_this_month/100.0, 0) as net_transaction_amount_this_month, - coalesce(transactions_grouped.sales_count_this_month, 0) as sales_count_this_month, - coalesce(transactions_grouped.refund_count_this_month, 0) as refund_count_this_month, - transactions_grouped.first_sale_date, - transactions_grouped.most_recent_sale_date, + customers.customer_email, + customers.created_at as customer_created_at, + customers.is_delinquent, + coalesce(transactions_by_customer.total_sales/100.0, 0) as total_sales, + coalesce(transactions_by_customer.total_refunds/100.0, 0) as total_refunds, + coalesce(transactions_by_customer.total_gross_transaction_amount/100.0, 0) as total_gross_transaction_amount, + coalesce(transactions_by_customer.total_fees/100.0, 0) as total_fees, + coalesce(transactions_by_customer.total_net_transaction_amount/100.0, 0) as total_net_transaction_amount, + coalesce(transactions_by_customer.total_sales_count, 0) as total_sales_count, + coalesce(transactions_by_customer.total_refund_count, 0) as total_refund_count, + coalesce(transactions_by_customer.sales_this_month/100.0, 0) as sales_this_month, + coalesce(transactions_by_customer.refunds_this_month/100.0, 0) as refunds_this_month, + coalesce(transactions_by_customer.gross_transaction_amount_this_month/100.0, 0) as gross_transaction_amount_this_month, + coalesce(transactions_by_customer.fees_this_month/100.0, 0) as fees_this_month, + coalesce(transactions_by_customer.net_transaction_amount_this_month/100.0, 0) as net_transaction_amount_this_month, + coalesce(transactions_by_customer.sales_count_this_month, 0) as sales_count_this_month, + coalesce(transactions_by_customer.refund_count_this_month, 0) as refund_count_this_month, + transactions_by_customer.first_sale_date, + transactions_by_customer.most_recent_sale_date, 0 as total_failed_charge_count, 0 as total_failed_charge_amount, 0 as failed_charge_count_this_month, 0 as failed_charge_amount_this_month, - customer.currency as customer_currency, - customer.default_card_id, - customer.shipping_name, - customer.shipping_address_line_1, - customer.shipping_address_line_2, - customer.shipping_address_city, - customer.shipping_address_state, - customer.shipping_address_country, - customer.shipping_address_postal_code, - customer.shipping_phone + customers.customer_currency, + customers.default_card_id, + customers.shipping_name, + customers.shipping_address_line_1, + customers.shipping_address_line_2, + customers.shipping_address_city, + customers.shipping_address_state, + customers.shipping_address_country, + customers.shipping_address_postal_code, + customers.phone from transactions_by_customer left join customers using(customer_id) - where customer.customer_id is null and customer.description is null + where customers.customer_id is null and customers.customer_description is null ), customer_transactions_overview as ( select - coalesce(customer.description, customer.customer_id) as customer_description, - customer.email, - customer.created_at as customer_created_at, - customer.is_delinquent, - coalesce(transactions_grouped.total_sales/100.0, 0) as total_sales, - coalesce(transactions_grouped.total_refunds/100.0, 0) as total_refunds, - coalesce(transactions_grouped.total_gross_transaction_amount/100.0, 0) as total_gross_transaction_amount, - coalesce(transactions_grouped.total_fees/100.0, 0) as total_fees, - coalesce(transactions_grouped.total_net_transaction_amount/100.0, 0) as total_net_transaction_amount, - coalesce(transactions_grouped.total_sales_count, 0) as total_sales_count, - coalesce(transactions_grouped.total_refund_count, 0) as total_refund_count, - coalesce(transactions_grouped.sales_this_month/100.0, 0) as sales_this_month, - coalesce(transactions_grouped.refunds_this_month/100.0, 0) as refunds_this_month, - coalesce(transactions_grouped.gross_transaction_amount_this_month/100.0, 0) as gross_transaction_amount_this_month, - coalesce(transactions_grouped.fees_this_month/100.0, 0) as fees_this_month, - coalesce(transactions_grouped.net_transaction_amount_this_month/100.0, 0) as net_transaction_amount_this_month, - coalesce(transactions_grouped.sales_count_this_month, 0) as sales_count_this_month, - coalesce(transactions_grouped.refund_count_this_month, 0) as refund_count_this_month, - transactions_grouped.first_sale_date, - transactions_grouped.most_recent_sale_date, + coalesce(customers.customer_description, customers.customer_id) as customer_description, + customers.customer_email, + customers.created_at as customer_created_at, + customers.is_delinquent, + coalesce(transactions_by_customer.total_sales/100.0, 0) as total_sales, + coalesce(transactions_by_customer.total_refunds/100.0, 0) as total_refunds, + coalesce(transactions_by_customer.total_gross_transaction_amount/100.0, 0) as total_gross_transaction_amount, + coalesce(transactions_by_customer.total_fees/100.0, 0) as total_fees, + coalesce(transactions_by_customer.total_net_transaction_amount/100.0, 0) as total_net_transaction_amount, + coalesce(transactions_by_customer.total_sales_count, 0) as total_sales_count, + coalesce(transactions_by_customer.total_refund_count, 0) as total_refund_count, + coalesce(transactions_by_customer.sales_this_month/100.0, 0) as sales_this_month, + coalesce(transactions_by_customer.refunds_this_month/100.0, 0) as refunds_this_month, + coalesce(transactions_by_customer.gross_transaction_amount_this_month/100.0, 0) as gross_transaction_amount_this_month, + coalesce(transactions_by_customer.fees_this_month/100.0, 0) as fees_this_month, + coalesce(transactions_by_customer.net_transaction_amount_this_month/100.0, 0) as net_transaction_amount_this_month, + coalesce(transactions_by_customer.sales_count_this_month, 0) as sales_count_this_month, + coalesce(transactions_by_customer.refund_count_this_month, 0) as refund_count_this_month, + transactions_by_customer.first_sale_date, + transactions_by_customer.most_recent_sale_date, coalesce(failed_charges_by_customer.total_failed_charge_count, 0) as total_failed_charge_count, coalesce(failed_charges_by_customer.total_failed_charge_amount/100, 0) as total_failed_charge_amount, coalesce(failed_charges_by_customer.failed_charge_count_this_month, 0) as failed_charge_count_this_month, coalesce(failed_charges_by_customer.failed_charge_amount_this_month/100, 0) as failed_charge_amount_this_month, - customer.currency as customer_currency, - customer.default_card_id, - customer.shipping_name, - customer.shipping_address_line_1, - customer.shipping_address_line_2, - customer.shipping_address_city, - customer.shipping_address_state, - customer.shipping_address_country, - customer.shipping_address_postal_code, - customer.shipping_phone - from customer + customers.customer_currency, + customers.default_card_id, + customers.shipping_name, + customers.shipping_address_line_1, + customers.shipping_address_line_2, + customers.shipping_address_city, + customers.shipping_address_state, + customers.shipping_address_country, + customers.shipping_address_postal_code, + customers.phone + from customers left join transactions_by_customer using(customer_id) left join failed_charges_by_customer diff --git a/models/tmp/stg_stripe__customers.sql b/models/tmp/stg_stripe__customers.sql index 01167a5..29f1d4f 100644 --- a/models/tmp/stg_stripe__customers.sql +++ b/models/tmp/stg_stripe__customers.sql @@ -1,5 +1,31 @@ +with customer_addresses as ( + select + _airbyte_customers_hashid, + line1, + line2, + city, + state, + country, + postal_code + from {{ var('customers_address') }} +) + select id as customer_id, + {{ dbt_date.from_unixtimestamp('created') }} as created_at, description as customer_description, - email as customer_email -from {{ var('customers')}} \ No newline at end of file + email as customer_email, + delinquent as is_delinquent, + currency as customer_currency, + default_card as default_card_id, + shipping as shipping_name, + line1 as shipping_address_line_1, + line2 as shipping_address_line_2, + city as shipping_address_city, + state as shipping_address_state, + country as shipping_address_country, + postal_code as shipping_address_postal_code, + phone +from {{ var('customers')}} +left join customer_addresses + using(_airbyte_customers_hashid) \ No newline at end of file From c7256688b53fc2e875847d003fba4206b4a6a672 Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Fri, 6 May 2022 03:33:35 +0000 Subject: [PATCH 12/29] add new fields to subscriptions stg table --- models/tmp/stg_stripe__subscriptions.sql | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/models/tmp/stg_stripe__subscriptions.sql b/models/tmp/stg_stripe__subscriptions.sql index 3eb77f1..82c3900 100644 --- a/models/tmp/stg_stripe__subscriptions.sql +++ b/models/tmp/stg_stripe__subscriptions.sql @@ -1,6 +1,15 @@ select id as subscription_id, - billing as subscription_billing, + customer as customer_id, + status, {{ dbt_date.from_unixtimestamp('start') }} as start_date, - {{ dbt_date.from_unixtimestamp('ended_at') }} as ended_at + {{ dbt_date.from_unixtimestamp('ended_at') }} as ended_at, + billing as subscription_billing, + {{ dbt_date.from_unixtimestamp('billing_cycle_anchor') }} as billing_cycle_anchor, + {{ dbt_date.from_unixtimestamp('canceled_at') }} as canceled_at, + {{ dbt_date.from_unixtimestamp('created') }} as created_at, + {{ dbt_date.from_unixtimestamp('current_period_start') }} as current_period_start, + {{ dbt_date.from_unixtimestamp('current_period_end') }} as current_period_end, + days_until_due, + cancel_at_period_end as is_cancel_at_period_end from {{ var('subscriptions') }} \ No newline at end of file From 882b6a504865f8f260edc811a432ff8d93921bfa Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Fri, 6 May 2022 03:33:47 +0000 Subject: [PATCH 13/29] subscriptions table --- models/stripe__subscriptions.sql | 99 ++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 models/stripe__subscriptions.sql diff --git a/models/stripe__subscriptions.sql b/models/stripe__subscriptions.sql new file mode 100644 index 0000000..436bf41 --- /dev/null +++ b/models/stripe__subscriptions.sql @@ -0,0 +1,99 @@ +with invoices as ( + select + * + from {{ ref('stg_stripe__invoices') }} +), + +charges as ( + select + * + from {{ ref('stg_stripe__charges') }} +), + +invoice_line_items as ( + select + * + from {{ ref('stg_stripe__invoice_line_items') }} + +), + +subscriptions as ( + select + * + from {{ ref('stg_stripe__subscriptions') }} + +), + +customers as ( + select + * + from {{ ref('stg_stripe__customers') }} + +), + +line_items_by_invoice as ( + select + invoices.invoice_id, + invoices.amount_due, + invoices.amount_paid, + invoices.amount_remaining, + invoices.created_at, + max(invoice_line_items.subscription_id) as subscription_id, + sum(invoice_line_items.line_item_amount) as total_item_amount, + count(distinct invoice_line_items.invoice_line_item_id) as number_line_items + from invoice_line_items + left join invoices + using(invoice_id) + {{ dbt_utils.group_by(5) }} +), + +invoice_stats_by_sub as ( + select + subscription_id, + count(distinct invoice_id) as number_invoices_generated, + sum(amount_due) as total_amount_billed, + sum(amount_paid) as total_amount_paid, + sum(amount_remaining) total_amount_remaining, + max(created_at) as most_recent_invoice_created_at, + avg(amount_due) as average_invoice_amount, + avg(total_item_amount) as average_line_item_amount, + avg(number_line_items) as avg_num_invoice_items + from line_items_by_invoice + {{ dbt_utils.group_by(1) }} +), + +subscription_stats as ( + select + subscriptions.subscription_id, + subscriptions.customer_id, + customers.customer_description, + customers.customer_email, + subscriptions.status, + subscriptions.start_date, + subscriptions.ended_at, + subscriptions.subscription_billing, + subscriptions.billing_cycle_anchor, + subscriptions.canceled_at, + subscriptions.created_at, + subscriptions.current_period_start, + subscriptions.current_period_end, + subscriptions.days_until_due, + subscriptions.is_cancel_at_period_end, + number_invoices_generated, + total_amount_billed, + total_amount_paid, + total_amount_remaining, + most_recent_invoice_created_at, + average_invoice_amount, + average_line_item_amount, + avg_num_invoice_items + from subscriptions + left join invoice_stats_by_sub + using(subscription_id) + left join customers + using(customer_id) +) + +select * from subscription_stats + + From 55a3c7ade99ceb0315c815fd1d71fe326da0b448 Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Fri, 6 May 2022 03:34:00 +0000 Subject: [PATCH 14/29] sub payment int view --- .../tmp/int_stripe__subscription_payments.sql | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 models/tmp/int_stripe__subscription_payments.sql diff --git a/models/tmp/int_stripe__subscription_payments.sql b/models/tmp/int_stripe__subscription_payments.sql new file mode 100644 index 0000000..53a48e9 --- /dev/null +++ b/models/tmp/int_stripe__subscription_payments.sql @@ -0,0 +1,33 @@ +with subscription_items as ( + select + subscription_id, + {{ dbt_utils.date_trunc("day", 'invoice_items.invoice_created_at') }} as created_date, + -- invoice_number is used to determine for the order in which the invoices for a subscription occur + -- This is useful when counting sources of new MRR + count(subscription_id) over (partition by subscription_id order by invoice_created_at asc) as invoice_number + from + {{ ref('stripe__invoice_line_items') }} invoice_items + where + subscription_id IS NOT NULL + group by + invoice_items.subscription_id, + invoice_items.invoice_created_at +), + +subscription_payments as ( + select + subscription_items.subscription_id, + subs.average_invoice_amount as average_revenue, + invoice_number, + {{ dbt_utils.date_trunc("day", 'subs.created_at') }} as date, + subs.canceled_at, + subs.customer_email, + subs.status + from + subscription_items + left join + {{ref('stripe__subscriptions')}} subs + using(subscription_id) +) + +select * from subscription_payments \ No newline at end of file From ffcaf3a6ed6c9c85b7e69bb64df857914d2da659 Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Fri, 6 May 2022 04:01:30 +0000 Subject: [PATCH 15/29] int view of daily customer stats --- models/stripe__subscriptions.sql | 2 + .../tmp/int_stripe__daily_customer_stats.sql | 69 +++++++++++++++++++ models/tmp/stg_stripe__subscriptions.sql | 2 + 3 files changed, 73 insertions(+) create mode 100644 models/tmp/int_stripe__daily_customer_stats.sql diff --git a/models/stripe__subscriptions.sql b/models/stripe__subscriptions.sql index 436bf41..4fa7fb1 100644 --- a/models/stripe__subscriptions.sql +++ b/models/stripe__subscriptions.sql @@ -77,6 +77,8 @@ subscription_stats as ( subscriptions.created_at, subscriptions.current_period_start, subscriptions.current_period_end, + subscriptions.trial_start, + subscriptions.trial_end, subscriptions.days_until_due, subscriptions.is_cancel_at_period_end, number_invoices_generated, diff --git a/models/tmp/int_stripe__daily_customer_stats.sql b/models/tmp/int_stripe__daily_customer_stats.sql new file mode 100644 index 0000000..c4c309f --- /dev/null +++ b/models/tmp/int_stripe__daily_customer_stats.sql @@ -0,0 +1,69 @@ +with customers as ( + select + {{ dbt_utils.date_trunc("day", 'first_sale_date') }} as date, + count(*) as new_customers, + count( + case when (is_delinquent = false and total_sales > 0) then + 1 + end) as active_new_customers + from + {{ ref('stripe__customers') }} + where + first_sale_date is not null + group by + {{ dbt_utils.date_trunc("day", 'first_sale_date') }} +), + +trials as ( + select + {{ dbt_utils.date_trunc("day", 'created_at') }} as date, + count( + case when (trial_start is not null) then + 1 + end) as trials, + count( + case when (trial_start is not null + and total_amount_billed > 0) then + 1 + end) as trials_converted_on_day + from + {{ ref('stripe__subscriptions') }} + group by + {{ dbt_utils.date_trunc("day", 'created_at') }} +), + +-- consider a trial converted on the trial end date +trial_conversions as ( + select + {{ dbt_utils.date_trunc("day", 'trial_end') }} as date, + count( + case when (trial_end is not null + and total_amount_billed > 0) then + 1 + end) as trials_converted + from + {{ ref('stripe__subscriptions') }} + group by + {{ dbt_utils.date_trunc("day", 'trial_end') }} +), + +customers_over_time as ( + select + daily_overview.date, + coalesce(customers.new_customers, 0) as new_customers, + coalesce(customers.active_new_customers, 0) as new_paying_customers, + sum(coalesce(customers.active_new_customers, 0)) over (order by daily_overview.date rows unbounded preceding) as active_customers, + coalesce(trials.trials, 0) as trials, + coalesce(trial_conversions.trials_converted, 0) as trials_converted, + coalesce(round(cast(trials.trials_converted_on_day as decimal)/nullif(trials.trials, 0), 2) * 100, 0) as trial_conversion_rate + from + {{ref('int_stripe__daily_transactions')}} daily_overview + left join customers + using(date) + left join trials + using(date) + left join trial_conversions + using(date) +) + +select * from customers_over_time \ No newline at end of file diff --git a/models/tmp/stg_stripe__subscriptions.sql b/models/tmp/stg_stripe__subscriptions.sql index 82c3900..238e4a9 100644 --- a/models/tmp/stg_stripe__subscriptions.sql +++ b/models/tmp/stg_stripe__subscriptions.sql @@ -10,6 +10,8 @@ select {{ dbt_date.from_unixtimestamp('created') }} as created_at, {{ dbt_date.from_unixtimestamp('current_period_start') }} as current_period_start, {{ dbt_date.from_unixtimestamp('current_period_end') }} as current_period_end, + {{ dbt_date.from_unixtimestamp('trial_start') }} as trial_start, + {{ dbt_date.from_unixtimestamp('trial_end') }} as trial_end, days_until_due, cancel_at_period_end as is_cancel_at_period_end from {{ var('subscriptions') }} \ No newline at end of file From 70093a7bee093908156cdb1ffa596d8404d5a0f4 Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Fri, 6 May 2022 04:12:38 +0000 Subject: [PATCH 16/29] change plan_is_active name --- models/schema.yml | 2 +- models/stripe__invoice_line_items.sql | 2 +- models/tmp/stg_stripe__plans.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/schema.yml b/models/schema.yml index 02a90c1..c02d31f 100644 --- a/models/schema.yml +++ b/models/schema.yml @@ -185,7 +185,7 @@ models: description: "The end date of the subscription purchase" - name: plan_id description: "The plan associated with the purchased subscription" - - name: plan_is_active + - name: is_plan_active description: "A flag for whether the subscription plan is active" - name: plan_amount description: "The total amount for the subscription plan" diff --git a/models/stripe__invoice_line_items.sql b/models/stripe__invoice_line_items.sql index 988ccd6..4e920cd 100644 --- a/models/stripe__invoice_line_items.sql +++ b/models/stripe__invoice_line_items.sql @@ -78,7 +78,7 @@ line_items_summary as ( subscriptions.start_date as subscription_start_date, subscriptions.ended_at as subscription_ended_at, plans.plan_id, - plans.plan_is_active, + plans.is_plan_active, plans.plan_amount, plans.plan_interval, plans.plan_interval_count, diff --git a/models/tmp/stg_stripe__plans.sql b/models/tmp/stg_stripe__plans.sql index 032e5d7..f362eb2 100644 --- a/models/tmp/stg_stripe__plans.sql +++ b/models/tmp/stg_stripe__plans.sql @@ -1,6 +1,6 @@ select id as plan_id, - active as plan_is_active, + active as is_plan_active, amount as plan_amount, interval as plan_interval, interval_count as plan_interval_count, From 4ece17b9b53fa030114b6682d839371ad5848f8e Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Sat, 7 May 2022 18:50:46 +0000 Subject: [PATCH 17/29] v1 daily overview --- models/stripe__daily_overview.sql | 248 ++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 models/stripe__daily_overview.sql diff --git a/models/stripe__daily_overview.sql b/models/stripe__daily_overview.sql new file mode 100644 index 0000000..877eef3 --- /dev/null +++ b/models/stripe__daily_overview.sql @@ -0,0 +1,248 @@ +with daily_transactions as ( + select + * + from ref('int_stripe__daily_transactions') +), + +subscription_payments as ( + select + * + from ref('int_stripe__subscription_payments') +), + +customer_stats as ( + select + * + from ref('int_stripe__daily_customer_stats.sql') +) + +sub_stats as ( + select + {{ dbt_utils.date_trunc("day", 'dt.date') }} as date, + ( + /* + Churned subscriptions are counted when the current day is + less than the day the subscription was canceled. + */ + select + count( + case when ( + filtered_subs.status = 'canceled' + and dt.date = date_trunc('day', filtered_subs.canceled_at) + and filtered_subs.customer_email is not null + ) then + 1 + end) as "churned_subscriptions" + from ( + select distinct on (sp.subscription_id) + subscription_payments.date, + subscription_id, + status, + customer_email, + canceled_at + from + subscription_payments + where + sp.date <= date_trunc('day', dt.date) + order by + subscription_id, + date desc + ) as filtered_subs + ), + ( + /* + New subscriptions are counted on the first issued invoice. + */ + select + count( + case when ( + ( + filtered_subs.status = 'active' + or filtered_subs.status = 'past due' + or filtered_subs.status = 'canceled' + ) + and filtered_subs.invoice_number = 1 + and filtered_subs.customer_email is not null + ) then + 1 + end) as "new_subscriptions" + from ( + select distinct on (sp.subscription_id) + subscription_payments.date, + subscription_id, + invoice_number, + status, + customer_email, + canceled_at + from + subscription_payments + where + sp.date = date_trunc('day', dt.date) + order by + subscription_id, + sp.date desc + ) as filtered_subs + ), + ( + /* + Active subscriptions are counted when the status is active or past due. + If the subscription is canceled, the sub will be counted only if the current + day is less than the day the sub was canceled. This does not apply to subs + that are 'set to cancel' the day they are canceled. + */ + select + count( + case when ( + ( + filtered_subs.status = 'active' + or filtered_subs.status = 'past due' + or ( + filtered_subs.status = 'canceled' + and (cancel_at is null or date_trunc('day', filtered_subs.canceled_at) < date_trunc('day', filtered_subs.cancel_at)) + and dt.date < date_trunc('day', filtered_subs.canceled_at) + ) + ) + and filtered_subs.customer_email is not null + ) then + 1 + end) as "active_subscriptions" + from ( + select distinct on (sp.subscription_id) + subscription_payments.date, + subscription_id, + status, + customer_email, + canceled_at, + cancel_at + from + subscription_payments + where + sp.date <= date_trunc('day', dt.date) + order by + subscription_id, + sp.date desc) as filtered_subs + ), + /* + There are multiple different values that can be used when summing mrr. Stripe + seems to use the average value of the subscription over its entire lifetime. + */ + ( + select + coalesce(round(sum( + case when ( + filtered_subs.status = 'canceled' + and dt.date = date_trunc('day', filtered_subs.canceled_at) + and filtered_subs.customer_email is not null + ) then + filtered_subs.average_revenue / 100 + end), 2), 0) as "churned_mrr" + from ( + select distinct on (sp.subscription_id) + subscription_payments.date, + subscription_id, + average_revenue, + status, + customer_email, + canceled_at + from + subscription_payments + where + sp.date <= date_trunc('day', dt.date) + order by + subscription_id, + sp.date desc + ) as filtered_subs + ), + ( + select + coalesce(round(sum( + case when ( + ( + filtered_subs.status = 'active' + or filtered_subs.status = 'past due' + or filtered_subs.status = 'canceled' + ) + and filtered_subs.invoice_number = 1 + and filtered_subs.customer_email is not null + ) then + filtered_subs.average_revenue / 100 + end), 2), 0) as "new_mrr" + from ( + select distinct on (sp.subscription_id) + subscription_payments.date, + subscription_id, + invoice_number, + average_revenue, + status, + customer_email + from + subscription_payments + where + sp.date = date_trunc('day', dt.date) + order by + subscription_id, + date desc + ) as filtered_subs + ), + ( + select + coalesce(round(sum( + case when ( + ( + filtered_subs.status = 'active' + or filtered_subs.status = 'past due' + or ( + filtered_subs.status = 'canceled' + and (cancel_at is null or date_trunc('day', filtered_subs.canceled_at) < date_trunc('day', filtered_subs.cancel_at)) + and dt.date < date_trunc('day', filtered_subs.canceled_at) + ) + ) + and filtered_subs.customer_email is not null + ) then + filtered_subs.average_revenue / 100 + end), 2), 0) as "mrr" + from ( + select distinct on (sp.subscription_id) + subscription_payments.date, + subscription_id, + average_revenue, + status, + customer_email, + canceled_at, + cancel_at + from + subscription_payments + where + sp.date <= date_trunc('day', dt.date) + order by + subscription_id, + date desc + ) as filtered_subs + ) + from ( + select + date + from + daily_transactions + ) as dt +), + +daily_overview as ( + select + *, + coalesce(round(mrr/nullif(active_subscriptions, 0), 2), 0) as mrr_per_subscription, + coalesce(round(mrr/nullif(active_customers, 0), 2), 0) as mrr_per_customer, + coalesce(active_customers - lag(active_customers, 1) over (order by date), 0) as customers_diff, + coalesce(active_subscriptions - lag(active_subscriptions, 1) over (order by date), 0) as subscriptions_diff, + coalesce(mrr - lag(mrr, 1) over (order by date), 0) as mrr_diff + from + daily_transactions + left join sub_stats + using(date) + left join customer_stats + using(date) + order by + date asc +) + +select * from daily_overview \ No newline at end of file From ba5e548186939ae0acf4488a6936d27e65dbe86b Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Mon, 9 May 2022 05:31:49 +0000 Subject: [PATCH 18/29] finish daily overview table --- models/stripe__daily_overview.sql | 49 ++++++++++++++----------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/models/stripe__daily_overview.sql b/models/stripe__daily_overview.sql index 877eef3..07fe22c 100644 --- a/models/stripe__daily_overview.sql +++ b/models/stripe__daily_overview.sql @@ -1,20 +1,20 @@ with daily_transactions as ( select * - from ref('int_stripe__daily_transactions') + from {{ ref('int_stripe__daily_transactions') }} ), subscription_payments as ( select * - from ref('int_stripe__subscription_payments') + from {{ ref('int_stripe__subscription_payments') }} ), customer_stats as ( select * - from ref('int_stripe__daily_customer_stats.sql') -) + from {{ ref('int_stripe__daily_customer_stats') }} +), sub_stats as ( select @@ -34,7 +34,7 @@ sub_stats as ( 1 end) as "churned_subscriptions" from ( - select distinct on (sp.subscription_id) + select distinct on (subscription_payments.subscription_id) subscription_payments.date, subscription_id, status, @@ -43,7 +43,7 @@ sub_stats as ( from subscription_payments where - sp.date <= date_trunc('day', dt.date) + subscription_payments.date <= date_trunc('day', dt.date) order by subscription_id, date desc @@ -67,7 +67,7 @@ sub_stats as ( 1 end) as "new_subscriptions" from ( - select distinct on (sp.subscription_id) + select distinct on (subscription_payments.subscription_id) subscription_payments.date, subscription_id, invoice_number, @@ -77,18 +77,17 @@ sub_stats as ( from subscription_payments where - sp.date = date_trunc('day', dt.date) + subscription_payments.date = date_trunc('day', dt.date) order by subscription_id, - sp.date desc + subscription_payments.date desc ) as filtered_subs ), ( /* Active subscriptions are counted when the status is active or past due. If the subscription is canceled, the sub will be counted only if the current - day is less than the day the sub was canceled. This does not apply to subs - that are 'set to cancel' the day they are canceled. + day is less than the day the sub was canceled. */ select count( @@ -98,7 +97,6 @@ sub_stats as ( or filtered_subs.status = 'past due' or ( filtered_subs.status = 'canceled' - and (cancel_at is null or date_trunc('day', filtered_subs.canceled_at) < date_trunc('day', filtered_subs.cancel_at)) and dt.date < date_trunc('day', filtered_subs.canceled_at) ) ) @@ -107,20 +105,19 @@ sub_stats as ( 1 end) as "active_subscriptions" from ( - select distinct on (sp.subscription_id) + select distinct on (subscription_payments.subscription_id) subscription_payments.date, subscription_id, status, customer_email, - canceled_at, - cancel_at + canceled_at from subscription_payments where - sp.date <= date_trunc('day', dt.date) + subscription_payments.date <= date_trunc('day', dt.date) order by subscription_id, - sp.date desc) as filtered_subs + subscription_payments.date desc) as filtered_subs ), /* There are multiple different values that can be used when summing mrr. Stripe @@ -137,7 +134,7 @@ sub_stats as ( filtered_subs.average_revenue / 100 end), 2), 0) as "churned_mrr" from ( - select distinct on (sp.subscription_id) + select distinct on (subscription_payments.subscription_id) subscription_payments.date, subscription_id, average_revenue, @@ -147,10 +144,10 @@ sub_stats as ( from subscription_payments where - sp.date <= date_trunc('day', dt.date) + subscription_payments.date <= date_trunc('day', dt.date) order by subscription_id, - sp.date desc + subscription_payments.date desc ) as filtered_subs ), ( @@ -168,7 +165,7 @@ sub_stats as ( filtered_subs.average_revenue / 100 end), 2), 0) as "new_mrr" from ( - select distinct on (sp.subscription_id) + select distinct on (subscription_payments.subscription_id) subscription_payments.date, subscription_id, invoice_number, @@ -178,7 +175,7 @@ sub_stats as ( from subscription_payments where - sp.date = date_trunc('day', dt.date) + subscription_payments.date = date_trunc('day', dt.date) order by subscription_id, date desc @@ -193,7 +190,6 @@ sub_stats as ( or filtered_subs.status = 'past due' or ( filtered_subs.status = 'canceled' - and (cancel_at is null or date_trunc('day', filtered_subs.canceled_at) < date_trunc('day', filtered_subs.cancel_at)) and dt.date < date_trunc('day', filtered_subs.canceled_at) ) ) @@ -202,18 +198,17 @@ sub_stats as ( filtered_subs.average_revenue / 100 end), 2), 0) as "mrr" from ( - select distinct on (sp.subscription_id) + select distinct on (subscription_payments.subscription_id) subscription_payments.date, subscription_id, average_revenue, status, customer_email, - canceled_at, - cancel_at + canceled_at from subscription_payments where - sp.date <= date_trunc('day', dt.date) + subscription_payments.date <= date_trunc('day', dt.date) order by subscription_id, date desc From 2974a45267041269bc5e60f7397c7b766670d52d Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Mon, 9 May 2022 05:35:58 +0000 Subject: [PATCH 19/29] round daily transaction table to 2 point --- models/tmp/int_stripe__daily_transactions.sql | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/models/tmp/int_stripe__daily_transactions.sql b/models/tmp/int_stripe__daily_transactions.sql index 507c845..ecb0f84 100644 --- a/models/tmp/int_stripe__daily_transactions.sql +++ b/models/tmp/int_stripe__daily_transactions.sql @@ -72,21 +72,21 @@ daily_failed_charges as ( daily_transactions as ( select daily_balance_transactions.date, - daily_balance_transactions.total_sales/100.0 as total_sales, - daily_balance_transactions.total_refunds/100.0 as total_refunds, - daily_balance_transactions.total_adjustments/100.0 as total_adjustments, - daily_balance_transactions.total_other_transactions/100.0 as total_other_transactions, - daily_balance_transactions.total_gross_transaction_amount/100.0 as total_gross_transaction_amount, - daily_balance_transactions.total_net_transactions/100.0 as total_net_transactions, - daily_balance_transactions.total_payout_fees/100.0 as total_payout_fees, - daily_balance_transactions.total_gross_payout_amount/100.0 as total_gross_payout_amount, - daily_balance_transactions.daily_net_activity/100.0 as daily_net_activity, - (daily_balance_transactions.daily_net_activity + daily_balance_transactions.total_gross_payout_amount)/100.0 as daily_end_balance, + round(daily_balance_transactions.total_sales/100.0, 2) as total_sales, + round(daily_balance_transactions.total_refunds/100.0, 2) as total_refunds, + round(daily_balance_transactions.total_adjustments/100.0, 2) as total_adjustments, + round(daily_balance_transactions.total_other_transactions/100.0, 2) as total_other_transactions, + round(daily_balance_transactions.total_gross_transaction_amount/100.0, 2) as total_gross_transaction_amount, + round(daily_balance_transactions.total_net_transactions/100.0, 2) as total_net_transactions, + round(daily_balance_transactions.total_payout_fees/100.0, 2) as total_payout_fees, + round(daily_balance_transactions.total_gross_payout_amount/100.0, 2) as total_gross_payout_amount, + round(daily_balance_transactions.daily_net_activity/100.0, 2) as daily_net_activity, + round((daily_balance_transactions.daily_net_activity + daily_balance_transactions.total_gross_payout_amount)/100.0, 2) as daily_end_balance, daily_balance_transactions.total_sales_count, daily_balance_transactions.total_payouts_count, daily_balance_transactions.total_adjustments_count, coalesce(daily_failed_charges.total_failed_charge_count, 0) as total_failed_charge_count, - coalesce(daily_failed_charges.total_failed_charge_amount/100, 0) as total_failed_charge_amount + round(coalesce(daily_failed_charges.total_failed_charge_amount/100, 0.0), 2) as total_failed_charge_amount from daily_balance_transactions left join daily_failed_charges using(date) From 6c2465ce7c18a597b628d9e68d443d2a102e9e9f Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Mon, 9 May 2022 05:50:06 +0000 Subject: [PATCH 20/29] add date series to daily transactions --- models/tmp/int_stripe__daily_transactions.sql | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/models/tmp/int_stripe__daily_transactions.sql b/models/tmp/int_stripe__daily_transactions.sql index ecb0f84..eea91e4 100644 --- a/models/tmp/int_stripe__daily_transactions.sql +++ b/models/tmp/int_stripe__daily_transactions.sql @@ -11,7 +11,7 @@ incomplete_charges as ( charge_amount from {{ ref('stg_stripe__charges')}} where not charge_is_captured -), +), daily_balance_transactions as ( select @@ -69,27 +69,47 @@ daily_failed_charges as ( {{ dbt_utils.group_by(1) }} ), +date_ranges as ( + select + {{ dbt_utils.date_trunc("day",'date') }} as min_date, + {{ dbt_utils.date_trunc("day", dbt_date.today()) }} as max_date + from + daily_balance_transactions + order by + date asc + limit 1 +), + +date_series AS ( + SELECT + generate_series(min_date, max_date, '1 day'::interval) AS date + FROM + date_ranges +), + daily_transactions as ( select daily_balance_transactions.date, - round(daily_balance_transactions.total_sales/100.0, 2) as total_sales, - round(daily_balance_transactions.total_refunds/100.0, 2) as total_refunds, - round(daily_balance_transactions.total_adjustments/100.0, 2) as total_adjustments, - round(daily_balance_transactions.total_other_transactions/100.0, 2) as total_other_transactions, - round(daily_balance_transactions.total_gross_transaction_amount/100.0, 2) as total_gross_transaction_amount, - round(daily_balance_transactions.total_net_transactions/100.0, 2) as total_net_transactions, - round(daily_balance_transactions.total_payout_fees/100.0, 2) as total_payout_fees, - round(daily_balance_transactions.total_gross_payout_amount/100.0, 2) as total_gross_payout_amount, - round(daily_balance_transactions.daily_net_activity/100.0, 2) as daily_net_activity, - round((daily_balance_transactions.daily_net_activity + daily_balance_transactions.total_gross_payout_amount)/100.0, 2) as daily_end_balance, - daily_balance_transactions.total_sales_count, - daily_balance_transactions.total_payouts_count, - daily_balance_transactions.total_adjustments_count, + round(coalesce(daily_balance_transactions.total_sales/100.0, 0), 2) as total_sales, + round(coalesce(daily_balance_transactions.total_refunds/100.0, 0), 2) as total_refunds, + round(coalesce(daily_balance_transactions.total_adjustments/100.0, 0), 2) as total_adjustments, + round(coalesce(daily_balance_transactions.total_other_transactions/100.0, 0), 2) as total_other_transactions, + round(coalesce(daily_balance_transactions.total_gross_transaction_amount/100.0, 0), 2) as total_gross_transaction_amount, + round(coalesce(daily_balance_transactions.total_net_transactions/100.0, 0), 2) as total_net_transactions, + round(coalesce(daily_balance_transactions.total_payout_fees/100.0, 0), 2) as total_payout_fees, + round(coalesce(daily_balance_transactions.total_gross_payout_amount/100.0, 0), 2) as total_gross_payout_amount, + round(coalesce(daily_balance_transactions.daily_net_activity/100.0, 0), 2) as daily_net_activity, + round(coalesce((daily_balance_transactions.daily_net_activity + daily_balance_transactions.total_gross_payout_amount)/100.0, 0), 2) as daily_end_balance, + coalesce(daily_balance_transactions.total_sales_count, 0) as total_sales_count, + coalesce(daily_balance_transactions.total_payouts_count, 0) total_payouts_count, + coalesce(daily_balance_transactions.total_adjustments_count, 0) as total_adjustments_count, coalesce(daily_failed_charges.total_failed_charge_count, 0) as total_failed_charge_count, round(coalesce(daily_failed_charges.total_failed_charge_amount/100, 0.0), 2) as total_failed_charge_amount from daily_balance_transactions left join daily_failed_charges using(date) + right join date_series + using(date) ) select * from daily_transactions \ No newline at end of file From fb76e6035d571a5074e14f3b6b988e4a8e9c78dd Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Mon, 9 May 2022 05:53:55 +0000 Subject: [PATCH 21/29] fix date column (only works for postgres) --- models/tmp/int_stripe__daily_transactions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/tmp/int_stripe__daily_transactions.sql b/models/tmp/int_stripe__daily_transactions.sql index eea91e4..2938d0f 100644 --- a/models/tmp/int_stripe__daily_transactions.sql +++ b/models/tmp/int_stripe__daily_transactions.sql @@ -89,7 +89,7 @@ date_series AS ( daily_transactions as ( select - daily_balance_transactions.date, + date_series.date, round(coalesce(daily_balance_transactions.total_sales/100.0, 0), 2) as total_sales, round(coalesce(daily_balance_transactions.total_refunds/100.0, 0), 2) as total_refunds, round(coalesce(daily_balance_transactions.total_adjustments/100.0, 0), 2) as total_adjustments, From 53d5681cea7829a14b82acb5ec6c1352154e36eb Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Mon, 9 May 2022 19:21:19 +0000 Subject: [PATCH 22/29] finalise schema file --- models/schema.yml | 278 +++++++++++++++++- models/stripe__customers.sql | 3 +- models/stripe__subscriptions.sql | 2 +- models/tmp/int_stripe__daily_transactions.sql | 19 +- 4 files changed, 289 insertions(+), 13 deletions(-) diff --git a/models/schema.yml b/models/schema.yml index c02d31f..4207950 100644 --- a/models/schema.yml +++ b/models/schema.yml @@ -23,11 +23,108 @@ models: - name: stripe__customers description: "A table storing all customer info" columns: - - name: id - description: "The unique identifier for the customer" + - name: customer_id + description: "The unique identifier of the customer" tests: - unique - not_null + - name: customer_description + description: "The unique identifier for the customer" + tests: + - not_null + - name: customer_created_at + description: "The date the customer was created" + tests: + - not_null + - name: total_sales + description: "The total amount of sales for the customer" + tests: + - not_null + - name: total_refunds + description: "The total amount of refunds to the customer" + tests: + - not_null + - name: total_gross_transaction_amount + description: "The total gross transaction amount for the customer" + tests: + - not_null + - name: total_fees + description: "The total amount fees generated by the customer" + tests: + - not_null + - name: total_net_transaction_amount + description: "The total net transaction amount for the customer" + tests: + - not_null + - name: total_failed_charge_amount + description: "The total amount of failed charges to the customer" + tests: + - not_null + - name: total_failed_charge_count + description: "The total number of failed charges to this customer" + tests: + - not_null + - name: total_sales_count + description: "The total number of sales made to the customer" + tests: + - not_null + - name: total_refund_count + description: "The total number of refunds made to the customer" + tests: + - not_null + - name: sales_this_month + description: "The total amount of sales made this month" + tests: + - not_null + - name: refunds_this_month + description: "The total amount of refunds made this month" + tests: + - not_null + - name: gross_transaction_amount_this_month + description: "The total gross transaction amount this month" + tests: + - not_null + - name: net_transaction_amount_this_month + description: "The total net transaction amount this month" + tests: + - not_null + - name: sales_count_this_month + description: "The total number of sales made this month" + tests: + - not_null + - name: refund_count_this_month + description: "The total number of refund_count made this month" + tests: + - not_null + - name: failed_charge_count_this_month + description: "The total number of failed_charge_count made this month" + tests: + - not_null + - name: failed_charge_amount_this_month + description: "The total number of failed_charge_amount made this month" + tests: + - not_null + - name: customer_currency + description: "The currency set for this currency" + - name: default_card_id + description: "The unique identifier of the default customer card" + - name: shipping_name + description: "The name of the shipping service" + - name: shipping_address_line_1 + description: "The 1st line of the customer shipping address" + - name: shipping_address_line_2 + description: "The 2nd line of the customer shipping address" + - name: shipping_address_city + description: "The city of the customer shipping address" + - name: shipping_address_state + description: "The state of the customer shipping address" + - name: shipping_address_country + description: "The country of the customer shipping address" + - name: shipping_address_postal_code + description: "The postal code of the customer shipping address" + - name: phone_number + description: "The phone number of the customer" + - name: stripe__balance_transactions description: "A table storing all balance transactions and associated charge data" columns: @@ -96,6 +193,7 @@ models: description: "The user specified source of the payout" - name: refund_reason description: "The refund reason (if applicable)" + - name: stripe__invoice_line_items description: "A table storing all line items associated with an invoice" columns: @@ -197,3 +295,179 @@ models: description: "The nickname of the subscription plan" - name: plan_product_id description: "The unique id of the product associated with subscription plan" + + - name: stripe__subscriptions + description: "A table storing the details of all subscription plans used by customers" + columns: + - name: subscription_id + description: "The unique identifier for the subscription" + tests: + - unique + - not_null + - name: customer_id + description: "The unique identifier for the customer" + tests: + - not_null + - name: customer_description + description: "The description of the customer" + tests: + - not_null + - name: customer_email + description: "The email of the customer" + tests: + - not_null + - name: status + description: "The status of the subscription" + tests: + - not_null + - name: start_date + description: "The start date of the subscription" + tests: + - not_null + - name: end_date + description: "The end date of the subscription" + tests: + - not_null + - name: subscription_billing + description: "The type of billing used for the subscription" + tests: + - not_null + - name: billing_cycle_anchor + description: "The billing cycle anchor for the subscription" + tests: + - not_null + - name: canceled_at + description: "The datetime the invoice was canceled" + - name: created_at + description: "The datetime the invoice was created" + tests: + - not_null + - name: current_period_start + description: "The datetime the current subscription period starts" + tests: + - not_null + - name: current_period_end + description: "The datetime the current subscription period ends" + tests: + - not_null + - name: trial_start + description: "The datetime the trial starts" + - name: trial_end + description: "The datetime the trial ends" + - name: days_until_due + description: "The number of days until the subscription id canceled" + - name: is_cancel_at_period_end + description: "A flag for whether the subscription will be cancled at the end of the period" + - name: number_invoices_generated + description: "The number of invoice generated for this subscription" + tests: + - not_null + - name: total_amount_billed + description: "The total amount billed for the subscription" + tests: + not_null + - name: total_amount_paid + description: "The total amount paid for the subscription" + tests: + not_null + - name: total_amount_remaining + description: "The total amount remaining for the subscription" + tests: + not_null + - name: most_recent_invoice_at + description: "The datetime the most recent invoice was generated" + - name: average_invoice_amount + description: "The average amount charged for the subscription" + tests: + - not_null + - name: average_line_item_amount + description: "The average amount of a line item which is bought as part of the subscription" + tests: + - not_null + - name: average_num_invoice_items + description: "The average amount of line items charged for a subscription" + tests: + - not_null + - name: "stripe__daily_overview" + description: "A table storing the daily overview statistics of your stripe account" + columns: + - name: date + description: "The date for the Stripe record" + tests: + - unique + - not_null + - name: total_sales + description: "The total amount of sales done on the date" + tests: + - not_null + - name: total_refunds + description: "The total amount of refunds done on the date" + tests: + - not_null + - name: total_adjustments + description: "The total amount of adjustments done on the date" + tests: + - not_null + - name: total_other_transactions + description: "The total amount of other transactions done on the date" + tests: + - not_null + - name: total_gross_transaction_amount + description: "The total amount of gross_transaction_amount done on the date" + tests: + - not_null + - name: total_payout_fees + description: "The total amount of payout_fees done on the date" + tests: + - not_null + - name: total_gross_payment_amount + description: "The total amount of gross_payment_amount done on the date" + tests: + - not_null + - name: total_failed_charge_amount + description: "The total amount of failed charges on the date" + tests: + - not_null + - name: total_sales_count + description: "The total number of sales done on the date" + tests: + - not_null + - name: total_payouts_count + description: "The total number of payouts done on the date" + tests: + - not_null + - name: total_adjustments_count + description: "The total number of adjustments done on the date" + tests: + - not_null + - name: total_failed_charge_count + description: "The total number of failed charges on the date" + tests: + - not_null + - name: daily_end_balance + description: "The daily end balance on the date" + tests: + - not_null + - name: churned_subscriptions + description: "The number of subscriptions that have been churned on the date" + - name: new_subscriptions + description: "The number of new subscriptions on the date" + - name: active_subscriptions + description: "The snapshot of the total number of subscriptions that were active at the date" + - name: churned_mrr + description: "The amount of monthly recurring revenue churned on that date" + - name: new_mrr + description: "The amount of new monthly recurring revenue gained on the date" + - name: mrr + description: "The snapshot of the total monthly recurring revenue on the date" + - name: mrr_per_subscription + description: "The monthly recurring revenue per active subscription" + - name: mrr_per_customer + description: "The monthly recurring revenue per active customer" + - name: customers_diff + description: "The change in number of customers from the previous date to the current date" + - name: subscriptions_diff + description: "The change in number of subscriptions from the previous date to the current date" + - name: mrr_diff + description: "The change in the number of customers from the previous date to the current date" + diff --git a/models/stripe__customers.sql b/models/stripe__customers.sql index 45abff2..ce8bf95 100644 --- a/models/stripe__customers.sql +++ b/models/stripe__customers.sql @@ -143,6 +143,7 @@ no_customer_transactions_overview as ( customer_transactions_overview as ( select + customer_id, coalesce(customers.customer_description, customers.customer_id) as customer_description, customers.customer_email, customers.created_at as customer_created_at, @@ -176,7 +177,7 @@ customer_transactions_overview as ( customers.shipping_address_state, customers.shipping_address_country, customers.shipping_address_postal_code, - customers.phone + customers.phone as phone_number from customers left join transactions_by_customer using(customer_id) diff --git a/models/stripe__subscriptions.sql b/models/stripe__subscriptions.sql index 4fa7fb1..d009121 100644 --- a/models/stripe__subscriptions.sql +++ b/models/stripe__subscriptions.sql @@ -88,7 +88,7 @@ subscription_stats as ( most_recent_invoice_created_at, average_invoice_amount, average_line_item_amount, - avg_num_invoice_items + average_num_invoice_items from subscriptions left join invoice_stats_by_sub using(subscription_id) diff --git a/models/tmp/int_stripe__daily_transactions.sql b/models/tmp/int_stripe__daily_transactions.sql index 2938d0f..af17a6d 100644 --- a/models/tmp/int_stripe__daily_transactions.sql +++ b/models/tmp/int_stripe__daily_transactions.sql @@ -69,10 +69,10 @@ daily_failed_charges as ( {{ dbt_utils.group_by(1) }} ), -date_ranges as ( +date_range as ( select - {{ dbt_utils.date_trunc("day",'date') }} as min_date, - {{ dbt_utils.date_trunc("day", dbt_date.today()) }} as max_date + cast({{ dbt_utils.date_trunc("day",'date') }} as date) as min_date, + cast({{ dbt_utils.date_trunc("day", dbt_date.today()) }} as date) as max_date from daily_balance_transactions order by @@ -80,11 +80,12 @@ date_ranges as ( limit 1 ), -date_series AS ( - SELECT - generate_series(min_date, max_date, '1 day'::interval) AS date - FROM - date_ranges + +date_spine as ( + select + {{ dbt_utils.date_spine(datepart="day", start_date="min_date", end_date="max_date") }} + from + date_range ), daily_transactions as ( @@ -108,7 +109,7 @@ daily_transactions as ( from daily_balance_transactions left join daily_failed_charges using(date) - right join date_series + right join date_spine using(date) ) From d85968f1288926f62e315b40112a2bcf0d9dc99f Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Mon, 9 May 2022 20:29:13 +0000 Subject: [PATCH 23/29] various fixes for schema.yml --- models/schema.yml | 179 +++++++++++++++---------------- models/stripe__customers.sql | 3 +- models/stripe__subscriptions.sql | 2 +- 3 files changed, 91 insertions(+), 93 deletions(-) diff --git a/models/schema.yml b/models/schema.yml index 4207950..9d1c6e9 100644 --- a/models/schema.yml +++ b/models/schema.yml @@ -124,7 +124,6 @@ models: description: "The postal code of the customer shipping address" - name: phone_number description: "The phone number of the customer" - - name: stripe__balance_transactions description: "A table storing all balance transactions and associated charge data" columns: @@ -193,7 +192,6 @@ models: description: "The user specified source of the payout" - name: refund_reason description: "The refund reason (if applicable)" - - name: stripe__invoice_line_items description: "A table storing all line items associated with an invoice" columns: @@ -295,12 +293,11 @@ models: description: "The nickname of the subscription plan" - name: plan_product_id description: "The unique id of the product associated with subscription plan" - - - name: stripe__subscriptions + - name: stripe__subscriptions description: "A table storing the details of all subscription plans used by customers" columns: - name: subscription_id - description: "The unique identifier for the subscription" + description: "The unique identifier for the subscription" tests: - unique - not_null @@ -365,15 +362,15 @@ models: - name: total_amount_billed description: "The total amount billed for the subscription" tests: - not_null + - not_null - name: total_amount_paid description: "The total amount paid for the subscription" tests: - not_null + - not_null - name: total_amount_remaining description: "The total amount remaining for the subscription" tests: - not_null + - not_null - name: most_recent_invoice_at description: "The datetime the most recent invoice was generated" - name: average_invoice_amount @@ -388,86 +385,86 @@ models: description: "The average amount of line items charged for a subscription" tests: - not_null - - name: "stripe__daily_overview" - description: "A table storing the daily overview statistics of your stripe account" - columns: - - name: date - description: "The date for the Stripe record" - tests: - - unique - - not_null - - name: total_sales - description: "The total amount of sales done on the date" - tests: - - not_null - - name: total_refunds - description: "The total amount of refunds done on the date" - tests: - - not_null - - name: total_adjustments - description: "The total amount of adjustments done on the date" - tests: - - not_null - - name: total_other_transactions - description: "The total amount of other transactions done on the date" - tests: - - not_null - - name: total_gross_transaction_amount - description: "The total amount of gross_transaction_amount done on the date" - tests: - - not_null - - name: total_payout_fees - description: "The total amount of payout_fees done on the date" - tests: - - not_null - - name: total_gross_payment_amount - description: "The total amount of gross_payment_amount done on the date" - tests: - - not_null - - name: total_failed_charge_amount - description: "The total amount of failed charges on the date" - tests: - - not_null - - name: total_sales_count - description: "The total number of sales done on the date" - tests: - - not_null - - name: total_payouts_count - description: "The total number of payouts done on the date" - tests: - - not_null - - name: total_adjustments_count - description: "The total number of adjustments done on the date" - tests: - - not_null - - name: total_failed_charge_count - description: "The total number of failed charges on the date" - tests: - - not_null - - name: daily_end_balance - description: "The daily end balance on the date" - tests: - - not_null - - name: churned_subscriptions - description: "The number of subscriptions that have been churned on the date" - - name: new_subscriptions - description: "The number of new subscriptions on the date" - - name: active_subscriptions - description: "The snapshot of the total number of subscriptions that were active at the date" - - name: churned_mrr - description: "The amount of monthly recurring revenue churned on that date" - - name: new_mrr - description: "The amount of new monthly recurring revenue gained on the date" - - name: mrr - description: "The snapshot of the total monthly recurring revenue on the date" - - name: mrr_per_subscription - description: "The monthly recurring revenue per active subscription" - - name: mrr_per_customer - description: "The monthly recurring revenue per active customer" - - name: customers_diff - description: "The change in number of customers from the previous date to the current date" - - name: subscriptions_diff - description: "The change in number of subscriptions from the previous date to the current date" - - name: mrr_diff - description: "The change in the number of customers from the previous date to the current date" - + - name: "stripe__daily_overview" + description: "A table storing the daily overview statistics of your stripe account" + columns: + - name: date + description: "The date for the Stripe record" + tests: + - unique + - not_null + - name: total_sales + description: "The total amount of sales done on the date" + tests: + - not_null + - name: total_refunds + description: "The total amount of refunds done on the date" + tests: + - not_null + - name: total_adjustments + description: "The total amount of adjustments done on the date" + tests: + - not_null + - name: total_other_transactions + description: "The total amount of other transactions done on the date" + tests: + - not_null + - name: total_gross_transaction_amount + description: "The total amount of gross_transaction_amount done on the date" + tests: + - not_null + - name: total_payout_fees + description: "The total amount of payout_fees done on the date" + tests: + - not_null + - name: total_gross_payment_amount + description: "The total amount of gross_payment_amount done on the date" + tests: + - not_null + - name: total_failed_charge_amount + description: "The total amount of failed charges on the date" + tests: + - not_null + - name: total_sales_count + description: "The total number of sales done on the date" + tests: + - not_null + - name: total_payouts_count + description: "The total number of payouts done on the date" + tests: + - not_null + - name: total_adjustments_count + description: "The total number of adjustments done on the date" + tests: + - not_null + - name: total_failed_charge_count + description: "The total number of failed charges on the date" + tests: + - not_null + - name: daily_end_balance + description: "The daily end balance on the date" + tests: + - not_null + - name: churned_subscriptions + description: "The number of subscriptions that have been churned on the date" + - name: new_subscriptions + description: "The number of new subscriptions on the date" + - name: active_subscriptions + description: "The snapshot of the total number of subscriptions that were active at the date" + - name: churned_mrr + description: "The amount of monthly recurring revenue churned on that date" + - name: new_mrr + description: "The amount of new monthly recurring revenue gained on the date" + - name: mrr + description: "The snapshot of the total monthly recurring revenue on the date" + - name: mrr_per_subscription + description: "The monthly recurring revenue per active subscription" + - name: mrr_per_customer + description: "The monthly recurring revenue per active customer" + - name: customers_diff + description: "The change in number of customers from the previous date to the current date" + - name: subscriptions_diff + description: "The change in number of subscriptions from the previous date to the current date" + - name: mrr_diff + description: "The change in the number of customers from the previous date to the current date" + diff --git a/models/stripe__customers.sql b/models/stripe__customers.sql index ce8bf95..d010914 100644 --- a/models/stripe__customers.sql +++ b/models/stripe__customers.sql @@ -101,6 +101,7 @@ failed_charges_by_customer as ( no_customer_transactions_overview as ( select + customer_id, 'No Associated Customer' as customer_description, customers.customer_email, customers.created_at as customer_created_at, @@ -134,7 +135,7 @@ no_customer_transactions_overview as ( customers.shipping_address_state, customers.shipping_address_country, customers.shipping_address_postal_code, - customers.phone + customers.phone as phone_number from transactions_by_customer left join customers using(customer_id) diff --git a/models/stripe__subscriptions.sql b/models/stripe__subscriptions.sql index d009121..47be96a 100644 --- a/models/stripe__subscriptions.sql +++ b/models/stripe__subscriptions.sql @@ -57,7 +57,7 @@ invoice_stats_by_sub as ( max(created_at) as most_recent_invoice_created_at, avg(amount_due) as average_invoice_amount, avg(total_item_amount) as average_line_item_amount, - avg(number_line_items) as avg_num_invoice_items + avg(number_line_items) as average_num_invoice_items from line_items_by_invoice {{ dbt_utils.group_by(1) }} ), From 4735de36ee00ecabf3b2cd60075b979ab5277cdf Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Mon, 9 May 2022 22:43:10 +0000 Subject: [PATCH 24/29] =?UTF-8?q?fix=20date=20spine.=20fix=20required=20va?= =?UTF-8?q?riable=20set=20to=20only=20be=20done=20=E2=80=9Cin=20execution?= =?UTF-8?q?=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/stripe__daily_overview.sql | 6 +-- models/tmp/int_stripe__daily_transactions.sql | 39 +++++++++++-------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/models/stripe__daily_overview.sql b/models/stripe__daily_overview.sql index 07fe22c..15e7b49 100644 --- a/models/stripe__daily_overview.sql +++ b/models/stripe__daily_overview.sql @@ -225,11 +225,11 @@ sub_stats as ( daily_overview as ( select *, - coalesce(round(mrr/nullif(active_subscriptions, 0), 2), 0) as mrr_per_subscription, - coalesce(round(mrr/nullif(active_customers, 0), 2), 0) as mrr_per_customer, + coalesce(round(mrr/nullif(active_subscriptions, 0.0), 2), 0.0) as mrr_per_subscription, + coalesce(round(mrr/nullif(active_customers, 0.0), 2), 0.0) as mrr_per_customer, coalesce(active_customers - lag(active_customers, 1) over (order by date), 0) as customers_diff, coalesce(active_subscriptions - lag(active_subscriptions, 1) over (order by date), 0) as subscriptions_diff, - coalesce(mrr - lag(mrr, 1) over (order by date), 0) as mrr_diff + coalesce(mrr - lag(mrr, 1) over (order by date), 0.0) as mrr_diff from daily_transactions left join sub_stats diff --git a/models/tmp/int_stripe__daily_transactions.sql b/models/tmp/int_stripe__daily_transactions.sql index af17a6d..aa37ca7 100644 --- a/models/tmp/int_stripe__daily_transactions.sql +++ b/models/tmp/int_stripe__daily_transactions.sql @@ -1,3 +1,22 @@ +{%- set date_range_query -%} + select + cast({{ dbt_utils.date_trunc("day",'created_at') }} as date) as min_date, + cast({{ dbt_utils.date_trunc("day", dbt_date.today()) }} as date) as max_date + from + {{ ref('stripe__balance_transactions') }} + order by + created_at asc + limit 1 +{%- endset -%} +{% set date_range = run_query(date_range_query) %} +-- The dbt parser will cause dbt run to fail as these variables are set dynamically. +-- We avoid this by only calling set "in execution" +{% if execute %} + {% set min_date = date_range.columns[0][0] %} + {% set max_date = date_range.columns[1][0] %} +{% endif %} + + with balance_transactions as ( select * @@ -69,28 +88,14 @@ daily_failed_charges as ( {{ dbt_utils.group_by(1) }} ), -date_range as ( - select - cast({{ dbt_utils.date_trunc("day",'date') }} as date) as min_date, - cast({{ dbt_utils.date_trunc("day", dbt_date.today()) }} as date) as max_date - from - daily_balance_transactions - order by - date asc - limit 1 -), - date_spine as ( - select - {{ dbt_utils.date_spine(datepart="day", start_date="min_date", end_date="max_date") }} - from - date_range + {{ dbt_date.get_base_dates(start_date=min_date, end_date=max_date) }} ), daily_transactions as ( select - date_series.date, + date_spine.date_day as date, round(coalesce(daily_balance_transactions.total_sales/100.0, 0), 2) as total_sales, round(coalesce(daily_balance_transactions.total_refunds/100.0, 0), 2) as total_refunds, round(coalesce(daily_balance_transactions.total_adjustments/100.0, 0), 2) as total_adjustments, @@ -110,7 +115,7 @@ daily_transactions as ( left join daily_failed_charges using(date) right join date_spine - using(date) + on daily_balance_transactions.date = date_spine.date_day ) select * from daily_transactions \ No newline at end of file From 1f71d940e695d86594bab486fd24fe30f6015af0 Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Tue, 10 May 2022 01:27:41 +0000 Subject: [PATCH 25/29] reformat case statements to be more readable --- models/stripe__balance_transactions.sql | 10 +- models/stripe__customers.sql | 109 +++++++++++++----- models/stripe__daily_overview.sql | 38 +++--- .../tmp/int_stripe__daily_customer_stats.sql | 32 ++--- models/tmp/int_stripe__daily_transactions.sql | 77 +++++++++---- 5 files changed, 177 insertions(+), 89 deletions(-) diff --git a/models/stripe__balance_transactions.sql b/models/stripe__balance_transactions.sql index c6f9375..6d61154 100644 --- a/models/stripe__balance_transactions.sql +++ b/models/stripe__balance_transactions.sql @@ -39,8 +39,14 @@ balance_transactions_summary as ( charges.card_brand, charges.card_funding, charges.card_country, - case when balance_transactions.type = 'charges' then charges.charge_amount end as charge_amount, - case when balance_transactions.type = 'charges' then charges.charge_currency end as charge_currency, + case when + balance_transactions.type = 'charges' + then charges.charge_amount end + as charge_amount, + case when + balance_transactions.type = 'charges' + then charges.charge_currency end + as charge_currency, -- Customer details charges.customer_id, diff --git a/models/stripe__customers.sql b/models/stripe__customers.sql index d010914..605acf1 100644 --- a/models/stripe__customers.sql +++ b/models/stripe__customers.sql @@ -22,61 +22,104 @@ customers as ( transactions_by_customer as ( select customer_id, - sum(case when type in ('charge', 'payment') + sum( + case when + type in ('charge', 'payment') then amount else 0 - end) as total_sales, - sum(case when type in ('payment_refund', 'refund') + end + ) as total_sales, + sum( + case when + type in ('payment_refund', 'refund') then amount else 0 - end) as total_refunds, + end + ) as total_refunds, sum(amount) as total_gross_transaction_amount, sum(fee) as total_fees, sum(net_balance_change) as total_net_transaction_amount, - sum(case when type in ('charge', 'payment') + sum( + case when + type in ('charge', 'payment') then 1 else 0 - end) as total_sales_count, - sum(case when type in ('payment_refund', 'refund') + end + ) as total_sales_count, + sum( + case when + type in ('payment_refund', 'refund') then 1 else 0 - end) as total_refund_count, - sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} + end + ) as total_refund_count, + sum( + case when + type in ('charge', 'payment') + and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then amount else 0 - end) as sales_this_month, - sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} + end + ) as sales_this_month, + sum( + case when + type in ('payment_refund', 'refund') + and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then amount else 0 - end) as refunds_this_month, - sum(case when {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} + end + ) as refunds_this_month, + sum( + case when + {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then amount else 0 - end) as gross_transaction_amount_this_month, - sum(case when {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} + end + ) as gross_transaction_amount_this_month, + sum( + case when + {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then fee else 0 - end) as fees_this_month, - sum(case when {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} + end + ) as fees_this_month, + sum( + case when + {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then net_balance_change else 0 - end) as net_transaction_amount_this_month, - sum(case when type in ('charge', 'payment') and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} + end + ) as net_transaction_amount_this_month, + sum( + case when + type in ('charge', 'payment') + and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then 1 else 0 - end) as sales_count_this_month, - sum(case when type in ('payment_refund', 'refund') and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} + end + ) as sales_count_this_month, + sum( + case when + type in ('payment_refund', 'refund') + and {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then 1 else 0 - end) as refund_count_this_month, - min(case when type in ('charge', 'payment') + end + ) as refund_count_this_month, + min( + case when + type in ('charge', 'payment') then {{ dbt_utils.date_trunc("day", 'created_at') }} else null - end) as first_sale_date, - max(case when type in ('charge', 'payment') + end + ) as first_sale_date, + max( + case when + type in ('charge', 'payment') then {{ dbt_utils.date_trunc("day", 'created_at') }} else null - end) as most_recent_sale_date + end + ) as most_recent_sale_date from balance_transactions where type in ('payment', 'charge', 'payment_refund', 'refund') {{ dbt_utils.group_by(1) }} @@ -87,14 +130,20 @@ failed_charges_by_customer as ( customer_id, count(*) as total_failed_charge_count, sum(charge_amount) as total_failed_charge_amount, - sum(case when {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} + sum( + case when + {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then 1 else 0 - end) as failed_charge_count_this_month, - sum(case when {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} + end + ) as failed_charge_count_this_month, + sum( + case when + {{ dbt_utils.date_trunc("month", 'created_at') }} = {{ dbt_utils.date_trunc('month', dbt_date.today()) }} then charge_amount else 0 - end) as failed_charge_amount_this_month + end + ) as failed_charge_amount_this_month from incomplete_charges {{ dbt_utils.group_by(1) }} ), diff --git a/models/stripe__daily_overview.sql b/models/stripe__daily_overview.sql index 15e7b49..6666a9e 100644 --- a/models/stripe__daily_overview.sql +++ b/models/stripe__daily_overview.sql @@ -27,12 +27,12 @@ sub_stats as ( select count( case when ( - filtered_subs.status = 'canceled' + filtered_subs.status = 'canceled' and and dt.date = date_trunc('day', filtered_subs.canceled_at) and filtered_subs.customer_email is not null - ) then - 1 - end) as "churned_subscriptions" + ) + then 1 end + ) as "churned_subscriptions" from ( select distinct on (subscription_payments.subscription_id) subscription_payments.date, @@ -63,9 +63,9 @@ sub_stats as ( ) and filtered_subs.invoice_number = 1 and filtered_subs.customer_email is not null - ) then - 1 - end) as "new_subscriptions" + ) + then 1 end + ) as "new_subscriptions" from ( select distinct on (subscription_payments.subscription_id) subscription_payments.date, @@ -101,9 +101,9 @@ sub_stats as ( ) ) and filtered_subs.customer_email is not null - ) then - 1 - end) as "active_subscriptions" + ) + then 1 end + ) as "active_subscriptions" from ( select distinct on (subscription_payments.subscription_id) subscription_payments.date, @@ -130,9 +130,9 @@ sub_stats as ( filtered_subs.status = 'canceled' and dt.date = date_trunc('day', filtered_subs.canceled_at) and filtered_subs.customer_email is not null - ) then - filtered_subs.average_revenue / 100 - end), 2), 0) as "churned_mrr" + ) + then filtered_subs.average_revenue / 100 end + ), 2), 0) as "churned_mrr" from ( select distinct on (subscription_payments.subscription_id) subscription_payments.date, @@ -161,9 +161,9 @@ sub_stats as ( ) and filtered_subs.invoice_number = 1 and filtered_subs.customer_email is not null - ) then - filtered_subs.average_revenue / 100 - end), 2), 0) as "new_mrr" + ) + then filtered_subs.average_revenue / 100 end + ), 2), 0) as "new_mrr" from ( select distinct on (subscription_payments.subscription_id) subscription_payments.date, @@ -194,9 +194,9 @@ sub_stats as ( ) ) and filtered_subs.customer_email is not null - ) then - filtered_subs.average_revenue / 100 - end), 2), 0) as "mrr" + ) + then filtered_subs.average_revenue / 100 end + ), 2), 0) as "mrr" from ( select distinct on (subscription_payments.subscription_id) subscription_payments.date, diff --git a/models/tmp/int_stripe__daily_customer_stats.sql b/models/tmp/int_stripe__daily_customer_stats.sql index c4c309f..0c199ab 100644 --- a/models/tmp/int_stripe__daily_customer_stats.sql +++ b/models/tmp/int_stripe__daily_customer_stats.sql @@ -3,9 +3,10 @@ with customers as ( {{ dbt_utils.date_trunc("day", 'first_sale_date') }} as date, count(*) as new_customers, count( - case when (is_delinquent = false and total_sales > 0) then - 1 - end) as active_new_customers + case when + is_delinquent = false and total_sales > 0 + then 1 end + ) as active_new_customers from {{ ref('stripe__customers') }} where @@ -18,14 +19,16 @@ trials as ( select {{ dbt_utils.date_trunc("day", 'created_at') }} as date, count( - case when (trial_start is not null) then - 1 - end) as trials, + case when + trial_start is not null + then 1 end + ) as trials, count( - case when (trial_start is not null - and total_amount_billed > 0) then - 1 - end) as trials_converted_on_day + case when + trial_start is not null + and total_amount_billed > 0 + then 1 end + ) as trials_converted_on_day from {{ ref('stripe__subscriptions') }} group by @@ -37,10 +40,11 @@ trial_conversions as ( select {{ dbt_utils.date_trunc("day", 'trial_end') }} as date, count( - case when (trial_end is not null - and total_amount_billed > 0) then - 1 - end) as trials_converted + case when + (trial_end is not null + and total_amount_billed > 0) + then 1 end + ) as trials_converted from {{ ref('stripe__subscriptions') }} group by diff --git a/models/tmp/int_stripe__daily_transactions.sql b/models/tmp/int_stripe__daily_transactions.sql index aa37ca7..8b7e4c4 100644 --- a/models/tmp/int_stripe__daily_transactions.sql +++ b/models/tmp/int_stripe__daily_transactions.sql @@ -39,42 +39,71 @@ daily_balance_transactions as ( then {{ dbt_utils.date_trunc("day", 'available_on') }} else {{ dbt_utils.date_trunc("day", 'created_at') }} end as date, - sum(case when type in ('charge', 'payment') + sum( + case when + type in ('charge', 'payment') then amount - else 0 end) as total_sales, - sum(case when type in ('payment_refund', 'refund') + else 0 end + ) as total_sales, + sum(case when + type in ('payment_refund', 'refund') then amount - else 0 end) as total_refunds, - sum(case when type = 'adjustment' + else 0 end + ) as total_refunds, + sum(case when + type = 'adjustment' then amount - else 0 end) as total_adjustments, - sum(case when type not in ('charge', 'payment', 'payment_refund', 'refund', 'adjustment', 'payout') and type not like '%transfer%' + else 0 end + ) as total_adjustments, + sum(case when + type not in ('charge', 'payment', 'payment_refund', 'refund', 'adjustment', 'payout') + and type not like '%transfer%' then amount - else 0 end) as total_other_transactions, - sum(case when type <> 'payout' and type not like '%transfer%' + else 0 end + ) as total_other_transactions, + sum(case when + type <> 'payout' + and type not like '%transfer%' then amount - else 0 end) as total_gross_transaction_amount, - sum(case when type <> 'payout' and type not like '%transfer%' + else 0 end + ) as total_gross_transaction_amount, + sum(case when + type <> 'payout' + and type not like '%transfer%' then net_balance_change - else 0 end) as total_net_transactions, - sum(case when type = 'payout' or type like '%transfer%' + else 0 end + ) as total_net_transactions, + sum(case when + type = 'payout' + or type like '%transfer%' then fee * -1.0 - else 0 end) as total_payout_fees, - sum(case when type = 'payout' or type like '%transfer%' + else 0 end + ) as total_payout_fees, + sum(case when + type = 'payout' or type like '%transfer%' then amount - else 0 end) as total_gross_payout_amount, - sum(case when type = 'payout' or type like '%transfer%' + else 0 end + ) as total_gross_payout_amount, + sum(case when + type = 'payout' or type like '%transfer%' then fee * -1.0 - else net_balance_change end) as daily_net_activity, - sum(case when type in ('payment', 'charge') + else net_balance_change end + ) as daily_net_activity, + sum(case when + type in ('payment', 'charge') then 1 - else 0 end) as total_sales_count, - sum(case when type = 'payout' + else 0 end + ) as total_sales_count, + sum(case when + type = 'payout' then 1 - else 0 end) as total_payouts_count, - count(distinct case when type = 'adjustment' + else 0 end + ) as total_payouts_count, + count(distinct case when + type = 'adjustment' then coalesce(source, payout_id) - else null end) as total_adjustments_count + else null end + ) as total_adjustments_count from balance_transactions {{ dbt_utils.group_by(1) }} ), From dd6212fb32c9dd58acec9642f3f733d0a56ea2c0 Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Tue, 10 May 2022 08:37:03 +0000 Subject: [PATCH 26/29] fix uppercase --- models/tmp/int_stripe__subscription_payments.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/tmp/int_stripe__subscription_payments.sql b/models/tmp/int_stripe__subscription_payments.sql index 53a48e9..94b4876 100644 --- a/models/tmp/int_stripe__subscription_payments.sql +++ b/models/tmp/int_stripe__subscription_payments.sql @@ -6,9 +6,9 @@ with subscription_items as ( -- This is useful when counting sources of new MRR count(subscription_id) over (partition by subscription_id order by invoice_created_at asc) as invoice_number from - {{ ref('stripe__invoice_line_items') }} invoice_items + {{ ref('stripe__invoice_line_items') }} where - subscription_id IS NOT NULL + subscription_id is not null group by invoice_items.subscription_id, invoice_items.invoice_created_at From d8497de7f96d4cede2482f0a9dea983762c61b8f Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Thu, 12 May 2022 08:54:08 +0000 Subject: [PATCH 27/29] change mrr calculation to use plan_amount --- models/stripe__daily_overview.sql | 14 +++--- models/stripe__invoice_line_items.sql | 2 +- .../tmp/int_stripe__subscription_payments.sql | 46 +++++++++++++++---- models/tmp/stg_stripe__charges.sql | 4 +- models/tmp/stg_stripe__invoice_line_items.sql | 2 +- 5 files changed, 49 insertions(+), 19 deletions(-) diff --git a/models/stripe__daily_overview.sql b/models/stripe__daily_overview.sql index 6666a9e..1862b44 100644 --- a/models/stripe__daily_overview.sql +++ b/models/stripe__daily_overview.sql @@ -27,7 +27,7 @@ sub_stats as ( select count( case when ( - filtered_subs.status = 'canceled' and + filtered_subs.status = 'canceled' and dt.date = date_trunc('day', filtered_subs.canceled_at) and filtered_subs.customer_email is not null ) @@ -131,13 +131,13 @@ sub_stats as ( and dt.date = date_trunc('day', filtered_subs.canceled_at) and filtered_subs.customer_email is not null ) - then filtered_subs.average_revenue / 100 end + then filtered_subs.plan_amount / 100 end ), 2), 0) as "churned_mrr" from ( select distinct on (subscription_payments.subscription_id) subscription_payments.date, subscription_id, - average_revenue, + plan_amount, status, customer_email, canceled_at @@ -162,14 +162,14 @@ sub_stats as ( and filtered_subs.invoice_number = 1 and filtered_subs.customer_email is not null ) - then filtered_subs.average_revenue / 100 end + then filtered_subs.plan_amount / 100 end ), 2), 0) as "new_mrr" from ( select distinct on (subscription_payments.subscription_id) subscription_payments.date, subscription_id, invoice_number, - average_revenue, + plan_amount, status, customer_email from @@ -195,13 +195,13 @@ sub_stats as ( ) and filtered_subs.customer_email is not null ) - then filtered_subs.average_revenue / 100 end + then filtered_subs.plan_amount / 100 end ), 2), 0) as "mrr" from ( select distinct on (subscription_payments.subscription_id) subscription_payments.date, subscription_id, - average_revenue, + plan_amount, status, customer_email, canceled_at diff --git a/models/stripe__invoice_line_items.sql b/models/stripe__invoice_line_items.sql index 4e920cd..ce63950 100644 --- a/models/stripe__invoice_line_items.sql +++ b/models/stripe__invoice_line_items.sql @@ -40,7 +40,7 @@ line_items_summary as ( select -- Invoices invoices.invoice_id, - invoices.invoice_number, + invoices.invoice_number as invoice_number, invoices.created_at as invoice_created_at, invoices.status, invoices.due_date, diff --git a/models/tmp/int_stripe__subscription_payments.sql b/models/tmp/int_stripe__subscription_payments.sql index 94b4876..fec4c7d 100644 --- a/models/tmp/int_stripe__subscription_payments.sql +++ b/models/tmp/int_stripe__subscription_payments.sql @@ -1,32 +1,62 @@ -with subscription_items as ( +with subscription_invoice_number as ( select subscription_id, - {{ dbt_utils.date_trunc("day", 'invoice_items.invoice_created_at') }} as created_date, + {{ dbt_utils.date_trunc("day", 'invoice_created_at') }} as created_date, -- invoice_number is used to determine for the order in which the invoices for a subscription occur - -- This is useful when counting sources of new MRR + -- This is useful when counting sources of new MRR. We use this as invoice number as it is cardinal. count(subscription_id) over (partition by subscription_id order by invoice_created_at asc) as invoice_number from {{ ref('stripe__invoice_line_items') }} where subscription_id is not null group by - invoice_items.subscription_id, - invoice_items.invoice_created_at + subscription_id, + invoice_created_at +), + +balance_transactions as( + select + * + from + {{ ref('stripe__balance_transactions') }} +), + +subscription_items as ( + select + sub_items.subscription_id, + subscription_invoice_number.invoice_number, + balance_transactions.amount, + balance_transactions.net_balance_change, + balance_transactions.exchange_rate, + sub_items.plan_amount, + from + {{ ref('stripe__invoice_line_items') }} sub_items + left join subscription_invoice_number + on subscription_invoice_number.subscription_id=sub_items.subscription_id + and created_date={{ dbt_utils.date_trunc("day", 'invoice_created_at') }} + left join balance_transactions + using(balance_transaction_id) + where + sub_items.subscription_id is not null ), subscription_payments as ( select subscription_items.subscription_id, - subs.average_invoice_amount as average_revenue, invoice_number, {{ dbt_utils.date_trunc("day", 'subs.created_at') }} as date, subs.canceled_at, subs.customer_email, - subs.status + subs.status, + subs.average_invoice_amount as average_revenue, + amount, + net_balance_change, + exchange_rate, + plan_amount, from subscription_items left join - {{ref('stripe__subscriptions')}} subs + {{ ref('stripe__subscriptions') }} subs using(subscription_id) ) diff --git a/models/tmp/stg_stripe__charges.sql b/models/tmp/stg_stripe__charges.sql index 55e438a..8872ec0 100644 --- a/models/tmp/stg_stripe__charges.sql +++ b/models/tmp/stg_stripe__charges.sql @@ -4,7 +4,7 @@ with charges_card as ( brand as card_brand, funding as card_funding, country as card_country - from {{ var('charges_card')}} + from {{ var('charges_card') }} ) select @@ -21,6 +21,6 @@ select card_brand, card_funding, card_country -from {{ var('charges')}} +from {{ var('charges') }} left join charges_card using(_airbyte_charges_hashid) \ No newline at end of file diff --git a/models/tmp/stg_stripe__invoice_line_items.sql b/models/tmp/stg_stripe__invoice_line_items.sql index c4b98da..59e7c03 100644 --- a/models/tmp/stg_stripe__invoice_line_items.sql +++ b/models/tmp/stg_stripe__invoice_line_items.sql @@ -15,7 +15,7 @@ invoice_line_items_plan as ( select id as invoice_line_item_id, - invoice as invoice_id, + invoice_id, subscription as subscription_id, plan_id, description as line_item_description, From 4dc9085c9a14878cb28fa6c88fa79732d5f5ccdd Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Thu, 12 May 2022 08:54:15 +0000 Subject: [PATCH 28/29] fix commas --- models/tmp/int_stripe__subscription_payments.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/tmp/int_stripe__subscription_payments.sql b/models/tmp/int_stripe__subscription_payments.sql index fec4c7d..0de4d5e 100644 --- a/models/tmp/int_stripe__subscription_payments.sql +++ b/models/tmp/int_stripe__subscription_payments.sql @@ -28,7 +28,7 @@ subscription_items as ( balance_transactions.amount, balance_transactions.net_balance_change, balance_transactions.exchange_rate, - sub_items.plan_amount, + sub_items.plan_amount from {{ ref('stripe__invoice_line_items') }} sub_items left join subscription_invoice_number @@ -52,7 +52,7 @@ subscription_payments as ( amount, net_balance_change, exchange_rate, - plan_amount, + plan_amount from subscription_items left join From ccdf07858b75b2f5626300a0406b07e4a802febd Mon Sep 17 00:00:00 2001 From: Elijah Roussos Date: Thu, 12 May 2022 17:01:08 +0000 Subject: [PATCH 29/29] add comment --- models/stripe__daily_overview.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/models/stripe__daily_overview.sql b/models/stripe__daily_overview.sql index 1862b44..6d48ce6 100644 --- a/models/stripe__daily_overview.sql +++ b/models/stripe__daily_overview.sql @@ -1,3 +1,15 @@ +/* +The following select statements aim to best recreate the value for MRR found +on the Stripe Dashboard. They work filtering the data into a specific range +and using only the data from that range to create active subscription and MRR +values. + +As a baseline filter, all subscriptions not linked to a customer are discarded. + +We select distinct subscriptions from subscription_payments, taking the latest invoice +from the date range specified. +*/ + with daily_transactions as ( select *