Skip to content

Commit

Permalink
revert to async for now and formatting
Browse files Browse the repository at this point in the history
  • Loading branch information
leej3 committed Aug 7, 2024
1 parent 3b8be19 commit 037bdcb
Showing 1 changed file with 6 additions and 4 deletions.
10 changes: 6 additions & 4 deletions scripts/invocation_upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@

import pandas as pd
import requests
from motor import MotorClient
from odmantic import SyncEngine

# from motor.motor_tornado import MotorClient
from motor.motor_asyncio import AsyncIOMotorClient
from pydantic import ValidationError

from osm.schemas import Client, Invocation, Work
Expand Down Expand Up @@ -77,9 +78,9 @@ def read_data(data_path: str):

async def upload_data(invocations: List[Invocation], mongo_uri: str, db_name: str):
"""upload invocations to MongoDB one after the other to prevent timeout"""
motor_client = MotorClient(mongo_uri)
motor_client = AsyncIOMotorClient(mongo_uri)
try:
engine = SyncEngine(client=motor_client, database=db_name)
engine = motor_client(client=motor_client, database=db_name)
engine.save_all(invocations)
except (TypeError, Exception) as e:
if isinstance(e, TypeError):
Expand Down Expand Up @@ -114,6 +115,7 @@ def main(data_path="all_indicators.feather"):
if transformed_pickle.exists():
df = pickle.loads(transformed_pickle.read_bytes())
else:
breakpoint()
df = read_data(data_path)
if not df.empty:
invocations = transform_data(df)
Expand Down

0 comments on commit 037bdcb

Please sign in to comment.