Skip to content

Commit

Permalink
add task to async request unsplash backend napi
Browse files Browse the repository at this point in the history
  • Loading branch information
m-p-esser committed Oct 3, 2023
1 parent b006ec4 commit 2327945
Showing 1 changed file with 28 additions and 0 deletions.
28 changes: 28 additions & 0 deletions src/prefect/generic_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,12 @@
import json
import math
import random
import time
from pprint import pformat
from tempfile import NamedTemporaryFile
from typing import Literal

import httpx
import pandas as pd
import requests
from fake_useragent import UserAgent
Expand Down Expand Up @@ -152,6 +155,31 @@ def request_unsplash_napi(
return response


@task
async def request_unsplash_napi_async(
endpoint: str, proxies: dict = None, headers: dict = None, params: dict = None
):
logger = get_run_logger()

async with httpx.AsyncClient(proxies=proxies, verify=False) as client:
BASE_URL = "https://unsplash.com/napi"
URI = BASE_URL + endpoint

sleep_seconds = random.randint(1, 3)
logger.info(f"Sleeping for {sleep_seconds} seconds...")
time.sleep(sleep_seconds)

logger.info(f"Requesting URI: {URI}")
response = await client.get(url=URI, params=params, headers=headers)

logger.info(f"Request headers: \n {pformat(dict(response.request.headers))}")
logger.info(f"Response headers: \n {pformat(dict(response.headers))}")

response.raise_for_status()

return response


@task(retries=3, retry_delay_seconds=10)
@timer
def parse_response(response: requests.Response) -> dict:
Expand Down

0 comments on commit 2327945

Please sign in to comment.