Skip to content

Commit

Permalink
examples: add plaid report based read side features
Browse files Browse the repository at this point in the history
  • Loading branch information
@nikhilgarg28
nikhilgarg28 committed Dec 5, 2024
1 parent 1981b9c commit 417a95d
Show file tree
Hide file tree
Showing 7 changed files with 37,054 additions and 1 deletion.
Empty file added examples/plaid/__init__.py
View file
Empty file.
12 changes: 12 additions & 0 deletions examples/plaid/commit.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
from features import PlaidReport

TOKEN = "9nWHJMVT6YyaImFotfdCzhzHD30WE3ywh3CQIF0_zUU"
URL = "https://hoang.aws.fennel.ai/"
BRANCH = "plaid_test"

if __name__ == "__main__":
from fennel.client import Client

client = Client(url=URL, token=TOKEN)
client.checkout(BRANCH, init=True)
client.commit("add plaid report features", featuresets=[PlaidReport])
96 changes: 96 additions & 0 deletions examples/plaid/features.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
from fennel.featuresets import featureset, feature as F
from fennel.lib import meta
from fennel.lib.schema import struct
from typing import Optional, List
from datetime import datetime
from fennel.expr import col, var, lit


__owner__ = "[email protected]"


@struct
class CreditCategory:
detailed: str
primary: str


@struct
class Transaction:
transaction_id: str
account_id: str
amount: float

account_owner: Optional[str]
check_number: Optional[str]
credit_category: Optional[CreditCategory]
date: Optional[datetime]
date_posted: Optional[datetime]
date_transacted: Optional[datetime]
iso_currency_code: Optional[str]
merchant_name: Optional[str]
original_description: Optional[str]
unofficial_currency_code: Optional[str]


@featureset
class PlaidReport:
payload: str
transactions: List[Transaction] = F(
col("payload")
.str.json_extract("$.report.items[0].accounts[0].transactions")
.fillnull("[]")
.str.parse(List[Transaction]),
version=3,
)

num_transactions: int = F(col("transactions").list.len())

ecommerce_txns: List[Transaction] = F(
col("transactions").list.filter(
"t",
lit("amazon|walmart|target|bestbuy").str.contains(
var("t").struct.get("merchant_name").fillnull("Unknown")
),
),
version=1,
)
num_ecomerce_txns: int = F(col("ecommerce_txns").list.len())
any_ecomerce_txn: bool = F(col("ecommerce_txns").list.len() > 0)
total_ecomerce_spend: float = F(
col("ecommerce_txns").list.map("t", var("t").struct.get("amount")).sum()
)

atm_txns: List[Transaction] = F(
col("transactions").list.filter(
"t",
var("t")
.struct.get("credit_category")
.struct.get("primary")
.fillnull("Unknown")
== lit("ATM"),
),
version=1,
)
num_atm_txns: int = F(col("atm_txns").list.len())
any_atm_txn: bool = F(col("atm_txns").list.len() > 0)
total_atm_spend: float = F(
col("atm_txns").list.map("t", var("t").struct.get("amount")).sum()
)

retail_txns: List[Transaction] = F(
col("transactions").list.filter(
"t",
var("t")
.struct.get("credit_category")
.struct.get("primary")
.fillnull("Unknown")
== lit("RETAIL"),
),
version=1,
)
num_retail_txns: int = F(col("retail_txns").list.len())
any_retail_txn: bool = F(col("retail_txns").list.len() > 0)
total_retail_spend: float = F(
col("retail_txns").list.map("t", var("t").struct.get("amount")).sum()
)
209 changes: 209 additions & 0 deletions examples/plaid/mini.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
{
"report": {
"date_generated": "2024-07-16T01:52:42.912331716Z",
"days_requested": 365,
"items": [
{
"accounts": [
{
"transactions": [
{
"account_id": "NZzx4oRPkAHzyRekpG4PTZkoGpNAR4uypaj1E",
"account_owner": null,
"amount": 37.07,
"check_number": null,
"credit_category": {
"detailed": "GENERAL_MERCHANDISE_ONLINE_MARKETPLACES",
"primary": "GENERAL_MERCHANDISE"
},
"date": "2024-07-12",
"date_posted": "2024-07-12T00:00:00Z",
"date_transacted": "2024-07-12",
"iso_currency_code": "USD",
"location": {
"address": null,
"city": null,
"country": null,
"lat": null,
"lon": null,
"postal_code": null,
"region": null,
"state": null,
"store_number": null,
"zip": null
},
"merchant_name": "Amazon",
"original_description": "AMZN Mktp US*11111111 Amzn.com/bill WA AM",
"pending": false,
"transaction_id": "XA7ZLy8rXzt7D3j9B6LMIgv5VxyQkAhbKjzmp",
"unofficial_currency_code": null
},
{
"account_id": "NZzx4oRPkAHzyRekpG4PTZkoGpNAR4uypaj1E",
"account_owner": null,
"amount": 51.61,
"check_number": null,
"credit_category": {
"detailed": "DINING_DINING",
"primary": "DINING"
},
"date": "2024-07-12",
"date_posted": "2024-07-12T00:00:00Z",
"date_transacted": "2024-07-12",
"iso_currency_code": "USD",
"location": {
"address": null,
"city": null,
"country": null,
"lat": null,
"lon": null,
"postal_code": null,
"region": null,
"state": null,
"store_number": null,
"zip": null
},
"merchant_name": "Domino's",
"original_description": "DOMINO's XXXX 111-222-3333",
"pending": false,
"transaction_id": "VEPeMbWqRluPVZLQX4MDUkKRw41Ljzf9gyLBW",
"unofficial_currency_code": null
},
{
"account_id": "NZzx4oRPkAHzyRekpG4PTZkoGpNAR4uypaj1E",
"account_owner": null,
"amount": 7.55,
"check_number": null,
"credit_category": {
"detailed": "GENERAL_MERCHANDISE_FURNITURE_AND_HARDWARE",
"primary": "GENERAL_MERCHANDISE"
},
"date": "2024-07-12",
"date_posted": "2024-07-12T00:00:00Z",
"date_transacted": "2024-07-12",
"iso_currency_code": "USD",
"location": {
"address": null,
"city": "Chicago",
"country": null,
"lat": null,
"lon": null,
"postal_code": null,
"region": null,
"state": null,
"store_number": null,
"zip": null
},
"merchant_name": "IKEA",
"original_description": "IKEA CHICAGO",
"pending": false,
"transaction_id": "6GQZARgvroCAE1eW5wpQT7w3oB6nvzi8DKMBa",
"unofficial_currency_code": null
},
{
"account_id": "NZzx4oRPkAHzyRekpG4PTZkoGpNAR4uypaj1E",
"account_owner": null,
"amount": 44.21,
"check_number": null,
"credit_category": {
"detailed": "DINING_DINING",
"primary": "DINING"
},
"date": "2024-07-12",
"date_posted": "2024-07-12T00:00:00Z",
"date_transacted": "2024-07-12",
"iso_currency_code": "USD",
"location": {
"address": null,
"city": null,
"country": null,
"lat": null,
"lon": null,
"postal_code": null,
"region": null,
"state": null,
"store_number": null,
"zip": null
},
"merchant_name": null,
"original_description": "POKE BROS * POKE BRO IL",
"pending": false,
"transaction_id": "RpdN7W8GmRSdjZB9Jm7ATj4M86vdnktapkrgL",
"unofficial_currency_code": null
},
{
"account_id": "NZzx4oRPkAHzyRekpG4PTZkoGpNAR4uypaj1E",
"account_owner": null,
"amount": 36.82,
"check_number": null,
"credit_category": {
"detailed": "GENERAL_MERCHANDISE_DISCOUNT_STORES",
"primary": "GENERAL_MERCHANDISE"
},
"date": "2024-07-13",
"date_posted": "2024-07-13T00:00:00Z",
"date_transacted": "2024-07-13",
"iso_currency_code": "USD",
"location": {
"address": null,
"city": null,
"country": null,
"lat": null,
"lon": null,
"postal_code": null,
"region": null,
"state": null,
"store_number": null,
"zip": null
},
"merchant_name": "Family Dollar",
"original_description": "FAMILY DOLLAR",
"pending": false,
"transaction_id": "5AeQWvo5KLtAD9wNL68PTdAgPE7VNWf5Kye1G",
"unofficial_currency_code": null
},
{
"account_id": "NZzx4oRPkAHzyRekpG4PTZkoGpNAR4uypaj1E",
"account_owner": null,
"amount": 54.24,
"check_number": null,
"credit_category": {
"detailed": "FOOD_RETAIL_GROCERIES",
"primary": "FOOD_RETAIL"
},
"date": "2024-07-15",
"date_posted": "2024-07-15T00:00:00Z",
"date_transacted": "2024-07-15",
"iso_currency_code": "USD",
"location": {
"address": null,
"city": "Portland",
"country": null,
"lat": null,
"lon": null,
"postal_code": null,
"region": "OR",
"state": "OR",
"store_number": "1111",
"zip": null
},
"merchant_name": "Safeway",
"original_description": "SAFEWAY #1111 PORTLAND OR 111111",
"pending": false,
"transaction_id": "P13aP8b7nmS3WQoxg1PMsdvMK679RNfo65B4G",
"unofficial_currency_code": null
}
]
}
],
"date_last_updated": "2024-07-16T01:52:42.912331716Z",
"institution_id": "ins_109512",
"institution_name": "Houndstooth Bank",
"item_id": "NZzx4oRPkAHzyRekpG4PTZkDNkQW93tWnyGeA"
}
],
"report_id": "f3bb434f-1c9b-4ef2-b76c-3d1fd08156ec"
},
"warnings": [],
"request_id": "FibfL8t3s71KJnj"
}
59 changes: 59 additions & 0 deletions examples/plaid/query.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
from fennel.client import Client
import threading
import pandas as pd
import time
import json
from typing import Any
from features import PlaidReport

TOKEN = "9nWHJMVT6YyaImFotfdCzhzHD30WE3ywh3CQIF0_zUU"
URL = "https://hoang.aws.fennel.ai/"
BRANCH = "plaid_test"


def read_json(path: str) -> Any:
with open(path, "r") as f:
return f.read()


def traffic(t: int):
client = Client(url=URL, token=TOKEN)
client.checkout(BRANCH)
payload = read_json("sample.json")
print(f"Thread {t} started")
n = 0
while True:
if n % 100 == 0:
print(f"Thread {t} processing {n} requests")
once(client, payload)
n += 1


def once(client: Client, payload: str):
start = time.time()
df = client.query(
inputs=[PlaidReport.payload],
outputs=[
PlaidReport.num_transactions,
PlaidReport.num_ecomerce_txns,
PlaidReport.total_ecomerce_spend,
PlaidReport.any_ecomerce_txn,
PlaidReport.num_atm_txns,
PlaidReport.total_atm_spend,
PlaidReport.any_atm_txn,
PlaidReport.num_retail_txns,
PlaidReport.total_retail_spend,
PlaidReport.any_retail_txn,
],
input_dataframe=pd.DataFrame({"PlaidReport.payload": [payload]}),
)
end = time.time()
print(df)
print(f"Time taken: {end - start} seconds")


if __name__ == "__main__":
client = Client(url=URL, token=TOKEN)
client.checkout(BRANCH)
payload = read_json("sample.json")
once(client, payload)
Loading

0 comments on commit 417a95d

Please sign in to comment.