Skip to content

Commit

Permalink
Merge pull request #244 from TogetherCrew/feat/analyzer-helpers-disco…
Browse files Browse the repository at this point in the history
…urse

Fix: wrong kwarg given for FetchPlatforms of discourse_analyzer_etl
  • Loading branch information
amindadgar authored Aug 7, 2024
2 parents 001fe84 + 76e76cc commit 1a95e04
Showing 1 changed file with 7 additions and 4 deletions.
11 changes: 7 additions & 4 deletions dags/discourse_analyzer_etl.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,12 @@ def discourse_etl_raw_data(
transformed_data = transformer.transform(
raw_data=extracted_data,
)

loader = LoadTransformedData(platform_id=platform_id)
loader.load(processed_data=transformed_data, recompute=recompute)
if len(transformed_data) != 0:
logging.info(f"Loading {len(transformed_data)} transformed document in db!")
loader = LoadTransformedData(platform_id=platform_id)
loader.load(processed_data=transformed_data, recompute=recompute)
else:
logging.warning("No new document to load for discourse!")

@task
def discourse_etl_raw_members(
Expand Down Expand Up @@ -173,7 +176,7 @@ def analyze_discourse(platform_processed: dict[str, str | bool]) -> None:
```
"""
logging.info(f"platform_processed: {platform_processed}")
fetcher = FetchPlatforms(plaform_name="discourse")
fetcher = FetchPlatforms(platform_name="discourse")
platform_id = platform_processed["platform_id"]
recompute = platform_processed["recompute"]

Expand Down

0 comments on commit 1a95e04

Please sign in to comment.