From fdcedda161f7772f6426afb89afeceba7571961f Mon Sep 17 00:00:00 2001 From: The many faced demon <154847721+themanyfaceddemon@users.noreply.github.com> Date: Sun, 27 Oct 2024 13:08:53 +0300 Subject: [PATCH] Auto cl update v2 (#684) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit no cl, no fun part 2 Не мытьём так катанием --- Tools/ADT/auto_cl.py | 54 +++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/Tools/ADT/auto_cl.py b/Tools/ADT/auto_cl.py index 370132fb8f8..f5182bb0197 100644 --- a/Tools/ADT/auto_cl.py +++ b/Tools/ADT/auto_cl.py @@ -10,12 +10,6 @@ CHANGELOG_PATH = FILE_PATH.parents[2] / "Resources" / "Changelog" / "1ChangelogADT.yml" OLD_CHANGELOG_PATH = FILE_PATH.parent / "cl_old.yml" -rate_limit_info = { - "limit": 5000, - "remaining": 5000, - "reset_time": datetime.now(timezone.utc), -} - class NoDatesSafeLoader(yaml.SafeLoader): @classmethod def remove_implicit_resolver(cls, tag_to_remove): @@ -45,30 +39,30 @@ def represent_multiline_string(dumper, data): MyDumper.add_representer(dict, represent_dict_compact) MyDumper.add_representer(str, represent_multiline_string) -async def handle_rate_limit(response): - global rate_limit_info - - rate_limit_info["limit"] = int(response.headers.get("x-ratelimit-limit", rate_limit_info["limit"])) - rate_limit_info["remaining"] = int(response.headers.get("x-ratelimit-remaining", rate_limit_info["remaining"])) - reset_timestamp = int(response.headers.get("x-ratelimit-reset", rate_limit_info["reset_time"].timestamp())) - rate_limit_info["reset_time"] = datetime.fromtimestamp(reset_timestamp, tz=timezone.utc) - - logging.info(f"Rate limit updated: {rate_limit_info['remaining']} remaining out of {rate_limit_info['limit']}.") - -async def check_rate_limit(): - global rate_limit_info - if rate_limit_info["remaining"] == 0: - reset_in = (rate_limit_info["reset_time"] - datetime.now(timezone.utc)).total_seconds() - logging.warning(f"Rate limit exceeded. Waiting {reset_in:.2f} seconds.") - await asyncio.sleep(max(reset_in, 1)) +async def check_rate_limit(session: aiohttp.ClientSession): + async with session.get("https://api.github.com/rate_limit") as response: + if response.status == 200: + data = await response.json() + limit = data["rate"]["limit"] + remaining = data["rate"]["remaining"] + reset_timestamp = data["rate"]["reset"] + reset_time = datetime.fromtimestamp(reset_timestamp, tz=timezone.utc) + logging.info(f"Rate limit checked: {remaining} remaining out of {limit}.") + + if remaining == 0: + reset_in = (reset_time - datetime.now(timezone.utc)).total_seconds() + logging.warning(f"Rate limit exceeded. Waiting {reset_in:.2f} seconds.") + await asyncio.sleep(max(reset_in, 1)) + + else: + logging.error(f"Error checking rate limit: {response.status}") + response.raise_for_status() -async def fetch_with_retry(session, url, retries=3): +async def fetch_with_retry(session: aiohttp.ClientSession, url, retries=3): for attempt in range(retries): - await check_rate_limit() + await check_rate_limit(session) async with session.get(url) as response: - await handle_rate_limit(response) - if response.status == 404: logging.info(f"{url} not found (404). Skipping.") return None @@ -76,16 +70,20 @@ async def fetch_with_retry(session, url, retries=3): elif response.status == 200: logging.info(f"Successfully fetched data from {url}.") return await response.json() + + if response.status in {403, 429}: + logging.warning(f"Received status {response.status} from {url}. Checking rate limit and retrying.") + continue logging.error(f"Error fetching {url}: {response.status}") response.raise_for_status() -async def get_latest_pr_number(session, repo): +async def get_latest_pr_number(session: aiohttp.ClientSession, repo): url = f"https://api.github.com/repos/{repo}/pulls?state=all&sort=created&direction=desc&per_page=1" pr_list = await fetch_with_retry(session, url) return pr_list[0]["number"] if pr_list else None -async def get_pr_info(session, repo, pr_number): +async def get_pr_info(session: aiohttp.ClientSession, repo, pr_number): url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}" return await fetch_with_retry(session, url)