diff --git a/dags/github/github_api_helpers/smart_proxy.py b/dags/github/github_api_helpers/smart_proxy.py index e1d2804d..2c94d742 100644 --- a/dags/github/github_api_helpers/smart_proxy.py +++ b/dags/github/github_api_helpers/smart_proxy.py @@ -1,26 +1,74 @@ import logging import random - import requests +from requests import Response + +class UniqueRandomNumbers: + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super(UniqueRandomNumbers, cls).__new__(cls) + cls._instance.original_numbers = list(range(20001, 29981)) + cls._instance.numbers = cls._instance.original_numbers.copy() + random.shuffle(cls._instance.numbers) + cls._instance.counter = 0 + return cls._instance + + def get_unique_number(self): + # ! This is a sample code to limit the number of requests per specific time period + # if self.counter >= 3: + # time.sleep(60) + # self.counter = 0 + + if not self.numbers: + self.reset() + + self.counter += 1 + return self.numbers.pop() + def reset(self): + self.numbers = self.original_numbers.copy() + random.shuffle(self.numbers) -def get(url: str, params=None): +def get(url: str, params=None) -> Response: """ - Sends a GET request With Smart Proxy. + Sends a GET request with Smart Proxy and retries up to 10 times if necessary. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary, list of tuples or bytes to send - in the query string for the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param params: (optional) Dictionary, list of tuples or bytes to send in the query string for the :class:`Request`. :return: :class:`Response ` object :rtype: requests.Response """ - random_port = random.randint(20001, 29980) - proxy_url = f"http://spusfxy185:TwinTwinTwin@eu.dc.smartproxy.com:{random_port}" - logging.info(f"Using proxy {proxy_url}") - proxies = { - "http": proxy_url, - "https": proxy_url, - } - return requests.get(url=url, params=params, proxies=proxies) + urn = UniqueRandomNumbers() + max_attempts = 10 + attempt = 0 + + while attempt < max_attempts: + random_port = urn.get_unique_number() + proxy_url = f"http://spusfxy185:TwinTwinTwin@eu.dc.smartproxy.com:{random_port}" + logging.info(f"Attempt {attempt + 1}: Using proxy {proxy_url}") + proxies = { + "http": proxy_url, + "https": proxy_url, + } + + try: + response = requests.get(url=url, params=params, proxies=proxies) + + if response.status_code == 200: + return response + else: + logging.error(f"Failed to get {url} with status code {response.status_code}") + logging.error(f"Response: {response.text}") + + except requests.exceptions.HTTPError as http_err: + logging.error(f'HTTP error occurred، Failed to get {url} with error: {http_err}') + except Exception as err: + logging.error(f'Some error occurred during getting {url}, error: {err}') + + attempt += 1 + + logging.error(f"All attempts failed for URL: {url}") + raise Exception(f"All attempts failed for URL: {url}") \ No newline at end of file