chore(data-warehouse): Remove limiting logic temporarily (#19992)

* connection errors * remove args * remove args
PostHog · Jan 26, 2024 · 1a185c6 · 1a185c6
1 parent 859b2bd
commit 1a185c6
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 13 deletions.
diff --git a/posthog/temporal/data_imports/external_data_job.py b/posthog/temporal/data_imports/external_data_job.py
@@ -168,7 +168,8 @@ async def run_external_data_job(inputs: ExternalDataJobInputs) -> None:
         if not stripe_secret_key:
             raise ValueError(f"Stripe secret key not found for job {model.id}")
         source = stripe_source(
-            api_key=stripe_secret_key, endpoints=tuple(inputs.schemas), job_id=str(model.id), team_id=inputs.team_id
+            api_key=stripe_secret_key,
+            endpoints=tuple(inputs.schemas),
         )
     elif model.pipeline.source_type == ExternalDataSource.Type.HUBSPOT:
         from posthog.temporal.data_imports.pipelines.hubspot.auth import refresh_access_token
@@ -185,8 +186,6 @@ async def run_external_data_job(inputs: ExternalDataJobInputs) -> None:
         source = hubspot(
             api_key=hubspot_access_code,
             refresh_token=refresh_token,
-            job_id=str(model.id),
-            team_id=inputs.team_id,
             endpoints=tuple(inputs.schemas),
         )
     elif model.pipeline.source_type == ExternalDataSource.Type.POSTGRES:

diff --git a/posthog/temporal/data_imports/pipelines/hubspot/__init__.py b/posthog/temporal/data_imports/pipelines/hubspot/__init__.py
@@ -50,8 +50,6 @@
 def hubspot(
     api_key: str,
     refresh_token: str,
-    job_id: str,
-    team_id: int,
     endpoints: Sequence[str] = ("companies", "contacts", "deals", "tickets", "quotes"),
     include_history: bool = False,
 ) -> Iterable[DltResource]:
@@ -93,12 +91,9 @@ def hubspot(
             include_history=include_history,
             props=DEFAULT_PROPS[endpoint],
             include_custom_props=True,
-            job_id=job_id,
-            team_id=team_id,
         )
 
 
-@limit_paginated_generator
 def crm_objects(
     object_type: str,
     api_key: str,

diff --git a/posthog/temporal/data_imports/pipelines/stripe/helpers.py b/posthog/temporal/data_imports/pipelines/stripe/helpers.py
@@ -7,7 +7,6 @@
 from dlt.common import pendulum
 from dlt.sources import DltResource
 from pendulum import DateTime
-from posthog.temporal.data_imports.pipelines.helpers import limit_paginated_generator
 
 stripe.api_version = "2022-11-15"
 
@@ -49,7 +48,6 @@ def stripe_get_data(
     return response
 
 
-@limit_paginated_generator
 def stripe_pagination(
     api_key: str,
     endpoint: str,
@@ -84,7 +82,7 @@ def stripe_pagination(
 
 @dlt.source(max_table_nesting=0)
 def stripe_source(
-    api_key: str, endpoints: Tuple[str, ...], job_id: str, team_id: int, starting_after: Optional[str] = None
+    api_key: str, endpoints: Tuple[str, ...], starting_after: Optional[str] = None
 ) -> Iterable[DltResource]:
     for endpoint in endpoints:
         yield dlt.resource(
@@ -94,7 +92,5 @@ def stripe_source(
         )(
             api_key=api_key,
             endpoint=endpoint,
-            job_id=job_id,
-            team_id=team_id,
             starting_after=starting_after,
         )