add proxy_type as parameter

src/prefect/ingest_photos_expanded_napi_bigquery.py

@@ -4,6 +4,7 @@
 import asyncio
 import datetime
 import faulthandler
+from typing import Literal
 
 from prefect_gcp.bigquery import bigquery_insert_stream, bigquery_query
 from prefect_gcp.credentials import GcpCredentials
@@ -114,7 +115,10 @@ def write_request_log_to_bigquery(
 @flow(timeout_seconds=120)  # Main Flow (1st level) # Main Flow (1st level)
 @timer
 def ingest_photos_expanded_napi_bigquery(
-    gcp_credential_block_name: str, batch_size: int = 30, total_record_size: int = 300
+    gcp_credential_block_name: str,
+    proxy_type: Literal["datacenter", "residential"],
+    batch_size: int = 30,
+    total_record_size: int = 300,
 ):
     """Flow to load editorial photo metadata from Unsplash and store them in Bigquery"""
 
@@ -152,7 +156,7 @@ def ingest_photos_expanded_napi_bigquery(
     )
 
     # Prepare Proxy and Useragent
-    proxies = prepare_proxy_adresses("residential")
+    proxies = prepare_proxy_adresses(proxy_type)
     proxies["http://"] = proxies["http"]
     proxies["https://"] = proxies["https"]
     proxies.pop("http")
@@ -250,6 +254,7 @@ def ingest_photos_expanded_napi_bigquery(
     faulthandler.dump_traceback_later(60)
     ingest_photos_expanded_napi_bigquery(
         gcp_credential_block_name="unsplash-photo-trends-deployment-sa",
+        proxy_type="datacenter",
         batch_size=30,
         total_record_size=300,
     )

src/prefect/ingest_photos_napi_gcs.py

@@ -6,6 +6,7 @@
 from datetime import timedelta
 from pprint import pformat
 from random import randint
+from typing import Literal
 
 from google.cloud import storage
 from prefect_gcp.bigquery import bigquery_query
@@ -202,6 +203,7 @@ def write_request_log_to_bigquery(
 def ingest_photos_napi_gcs(
     gcp_credential_block_name: str,
     per_page: int,
+    proxy_type: Literal["datacenter", "residential"],
 ):
     """Flow to load Editorial photos from Unsplash and store them in a Google Cloud Storage Bucket"""
 
@@ -249,7 +251,7 @@ def ingest_photos_napi_gcs(
         time.sleep(randint(1, 3))
 
         # Prepare Proxy and Useragent
-        proxies = prepare_proxy_adresses("residential")
+        proxies = prepare_proxy_adresses(proxy_type)
         useragent_string = create_random_ua_string()
         logger.info(f"Will be using '{useragent_string}' to make next requests")
         headers = {"User-Agent": useragent_string}  # Overwrite Useragent
@@ -295,4 +297,5 @@ def ingest_photos_napi_gcs(
     ingest_photos_napi_gcs(
         gcp_credential_block_name="unsplash-photo-trends-deployment-sa",
         per_page=30,
+        proxy_type="datacenter",
     )