From 23279454f4eda54f7d5573d4f939788156dcddd4 Mon Sep 17 00:00:00 2001 From: Marc-Philipp Esser Date: Tue, 3 Oct 2023 15:04:46 +0200 Subject: [PATCH] add task to async request unsplash backend napi --- src/prefect/generic_tasks.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/prefect/generic_tasks.py b/src/prefect/generic_tasks.py index 471bbc0..82acc67 100644 --- a/src/prefect/generic_tasks.py +++ b/src/prefect/generic_tasks.py @@ -4,9 +4,12 @@ import json import math import random +import time +from pprint import pformat from tempfile import NamedTemporaryFile from typing import Literal +import httpx import pandas as pd import requests from fake_useragent import UserAgent @@ -152,6 +155,31 @@ def request_unsplash_napi( return response +@task +async def request_unsplash_napi_async( + endpoint: str, proxies: dict = None, headers: dict = None, params: dict = None +): + logger = get_run_logger() + + async with httpx.AsyncClient(proxies=proxies, verify=False) as client: + BASE_URL = "https://unsplash.com/napi" + URI = BASE_URL + endpoint + + sleep_seconds = random.randint(1, 3) + logger.info(f"Sleeping for {sleep_seconds} seconds...") + time.sleep(sleep_seconds) + + logger.info(f"Requesting URI: {URI}") + response = await client.get(url=URI, params=params, headers=headers) + + logger.info(f"Request headers: \n {pformat(dict(response.request.headers))}") + logger.info(f"Response headers: \n {pformat(dict(response.headers))}") + + response.raise_for_status() + + return response + + @task(retries=3, retry_delay_seconds=10) @timer def parse_response(response: requests.Response) -> dict: