From 00e7fdf34e82b0a572d300d88635daaf92d5dd45 Mon Sep 17 00:00:00 2001 From: Mohammad Twin Date: Wed, 20 Dec 2023 14:15:58 +0400 Subject: [PATCH 1/5] feat: resend request if fails --- dags/github/github_api_helpers/smart_proxy.py | 76 +++++++++++++++---- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/dags/github/github_api_helpers/smart_proxy.py b/dags/github/github_api_helpers/smart_proxy.py index e1d2804d..2c94d742 100644 --- a/dags/github/github_api_helpers/smart_proxy.py +++ b/dags/github/github_api_helpers/smart_proxy.py @@ -1,26 +1,74 @@ import logging import random - import requests +from requests import Response + +class UniqueRandomNumbers: + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super(UniqueRandomNumbers, cls).__new__(cls) + cls._instance.original_numbers = list(range(20001, 29981)) + cls._instance.numbers = cls._instance.original_numbers.copy() + random.shuffle(cls._instance.numbers) + cls._instance.counter = 0 + return cls._instance + + def get_unique_number(self): + # ! This is a sample code to limit the number of requests per specific time period + # if self.counter >= 3: + # time.sleep(60) + # self.counter = 0 + + if not self.numbers: + self.reset() + + self.counter += 1 + return self.numbers.pop() + def reset(self): + self.numbers = self.original_numbers.copy() + random.shuffle(self.numbers) -def get(url: str, params=None): +def get(url: str, params=None) -> Response: """ - Sends a GET request With Smart Proxy. + Sends a GET request with Smart Proxy and retries up to 10 times if necessary. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary, list of tuples or bytes to send - in the query string for the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param params: (optional) Dictionary, list of tuples or bytes to send in the query string for the :class:`Request`. :return: :class:`Response ` object :rtype: requests.Response """ - random_port = random.randint(20001, 29980) - proxy_url = f"http://spusfxy185:TwinTwinTwin@eu.dc.smartproxy.com:{random_port}" - logging.info(f"Using proxy {proxy_url}") - proxies = { - "http": proxy_url, - "https": proxy_url, - } - return requests.get(url=url, params=params, proxies=proxies) + urn = UniqueRandomNumbers() + max_attempts = 10 + attempt = 0 + + while attempt < max_attempts: + random_port = urn.get_unique_number() + proxy_url = f"http://spusfxy185:TwinTwinTwin@eu.dc.smartproxy.com:{random_port}" + logging.info(f"Attempt {attempt + 1}: Using proxy {proxy_url}") + proxies = { + "http": proxy_url, + "https": proxy_url, + } + + try: + response = requests.get(url=url, params=params, proxies=proxies) + + if response.status_code == 200: + return response + else: + logging.error(f"Failed to get {url} with status code {response.status_code}") + logging.error(f"Response: {response.text}") + + except requests.exceptions.HTTPError as http_err: + logging.error(f'HTTP error occurred، Failed to get {url} with error: {http_err}') + except Exception as err: + logging.error(f'Some error occurred during getting {url}, error: {err}') + + attempt += 1 + + logging.error(f"All attempts failed for URL: {url}") + raise Exception(f"All attempts failed for URL: {url}") \ No newline at end of file From 66b3014be68f821f544acd75c7278498b5fc2ee3 Mon Sep 17 00:00:00 2001 From: Mohammad Twin Date: Wed, 20 Dec 2023 14:29:45 +0400 Subject: [PATCH 2/5] feat: aligned with lint codes --- dags/github/github_api_helpers/smart_proxy.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/dags/github/github_api_helpers/smart_proxy.py b/dags/github/github_api_helpers/smart_proxy.py index 2c94d742..1e90a197 100644 --- a/dags/github/github_api_helpers/smart_proxy.py +++ b/dags/github/github_api_helpers/smart_proxy.py @@ -3,6 +3,7 @@ import requests from requests import Response + class UniqueRandomNumbers: _instance = None @@ -31,6 +32,7 @@ def reset(self): self.numbers = self.original_numbers.copy() random.shuffle(self.numbers) + def get(url: str, params=None) -> Response: """ Sends a GET request with Smart Proxy and retries up to 10 times if necessary. @@ -60,15 +62,19 @@ def get(url: str, params=None) -> Response: if response.status_code == 200: return response else: - logging.error(f"Failed to get {url} with status code {response.status_code}") + logging.error( + f"Failed to get {url} with status code {response.status_code}" + ) logging.error(f"Response: {response.text}") - + except requests.exceptions.HTTPError as http_err: - logging.error(f'HTTP error occurred، Failed to get {url} with error: {http_err}') + logging.error( + f"HTTP error occurred، Failed to get {url} with error: {http_err}" + ) except Exception as err: - logging.error(f'Some error occurred during getting {url}, error: {err}') + logging.error(f"Some error occurred during getting {url}, error: {err}") attempt += 1 logging.error(f"All attempts failed for URL: {url}") - raise Exception(f"All attempts failed for URL: {url}") \ No newline at end of file + raise Exception(f"All attempts failed for URL: {url}") From c83d95a0f18d0b8fb52a7a25f7d92b5ef45ce76e Mon Sep 17 00:00:00 2001 From: Mohammad Twin Date: Wed, 20 Dec 2023 15:25:38 +0400 Subject: [PATCH 3/5] feat: aligned with super linter --- dags/github/github_api_helpers/smart_proxy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dags/github/github_api_helpers/smart_proxy.py b/dags/github/github_api_helpers/smart_proxy.py index 1e90a197..0e9ce372 100644 --- a/dags/github/github_api_helpers/smart_proxy.py +++ b/dags/github/github_api_helpers/smart_proxy.py @@ -1,5 +1,6 @@ import logging import random + import requests from requests import Response From 70c2864b9c477414903bb0df9af3b1e18b39461b Mon Sep 17 00:00:00 2001 From: Mohammad Twin Date: Wed, 20 Dec 2023 15:37:21 +0400 Subject: [PATCH 4/5] feat: class vers were moved to the __init__ function of class --- dags/github/github_api_helpers/smart_proxy.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dags/github/github_api_helpers/smart_proxy.py b/dags/github/github_api_helpers/smart_proxy.py index 0e9ce372..816e900a 100644 --- a/dags/github/github_api_helpers/smart_proxy.py +++ b/dags/github/github_api_helpers/smart_proxy.py @@ -11,11 +11,13 @@ class UniqueRandomNumbers: def __new__(cls): if cls._instance is None: cls._instance = super(UniqueRandomNumbers, cls).__new__(cls) - cls._instance.original_numbers = list(range(20001, 29981)) - cls._instance.numbers = cls._instance.original_numbers.copy() - random.shuffle(cls._instance.numbers) - cls._instance.counter = 0 return cls._instance + + def __init__(self): + self.original_numbers = list(range(20001, 29981)) + self.numbers = self.original_numbers.copy() + random.shuffle(self.numbers) + self.counter = 0 def get_unique_number(self): # ! This is a sample code to limit the number of requests per specific time period From 8284e04005198a96f2c7ae1f69323fdb2427f903 Mon Sep 17 00:00:00 2001 From: Mohammad Twin Date: Wed, 20 Dec 2023 19:09:27 +0400 Subject: [PATCH 5/5] feat: aligned with super linter --- dags/github/github_api_helpers/smart_proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/github/github_api_helpers/smart_proxy.py b/dags/github/github_api_helpers/smart_proxy.py index 816e900a..7d98d8b7 100644 --- a/dags/github/github_api_helpers/smart_proxy.py +++ b/dags/github/github_api_helpers/smart_proxy.py @@ -12,7 +12,7 @@ def __new__(cls): if cls._instance is None: cls._instance = super(UniqueRandomNumbers, cls).__new__(cls) return cls._instance - + def __init__(self): self.original_numbers = list(range(20001, 29981)) self.numbers = self.original_numbers.copy()