From 22e4181d7837c4ca6aba8bd8ea00600c4695b733 Mon Sep 17 00:00:00 2001 From: Tom Forbes Date: Sun, 20 Oct 2024 14:14:39 +0100 Subject: [PATCH] Run in parallel --- src/pypi_data/cli.py | 10 ++++++---- src/pypi_data/combine_parquet.py | 1 - 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/pypi_data/cli.py b/src/pypi_data/cli.py index bd0ff37b..b4146fec 100644 --- a/src/pypi_data/cli.py +++ b/src/pypi_data/cli.py @@ -143,10 +143,12 @@ async def _run(r: CodeRepository) -> CodeRepository | None: async with asyncio.TaskGroup() as tg: tasks = [tg.create_task(_run(repo)) for repo in repositories] - for res in tasks: - res = await res - if res is not None: - results.append(res) + + for res in tasks: + res = res.result() + if res is not None: + results.append(res) + log.info(f"Fetched {len(tasks)} repository indexes") return results diff --git a/src/pypi_data/combine_parquet.py b/src/pypi_data/combine_parquet.py index faa8e2b3..00e40947 100644 --- a/src/pypi_data/combine_parquet.py +++ b/src/pypi_data/combine_parquet.py @@ -57,7 +57,6 @@ async def fill_buffer( time_hashing_ns = 0 time_iterating_ns = 0 time_loading_ns = 0 - time_downloading_ns = 0 start_time_ns = time.perf_counter_ns()