From 8187e5725a0f2bf3f17379066912871f6327584a Mon Sep 17 00:00:00 2001 From: Tom Forbes Date: Sun, 20 Oct 2024 12:46:42 +0100 Subject: [PATCH] Remove memory map --- src/pypi_data/combine_parquet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pypi_data/combine_parquet.py b/src/pypi_data/combine_parquet.py index 0f80d444..326d61bc 100644 --- a/src/pypi_data/combine_parquet.py +++ b/src/pypi_data/combine_parquet.py @@ -66,7 +66,7 @@ async def fill_buffer( log.info(f"Failed to download {repo.dataset_url}") continue log.info(f"Downloaded, reading {path}") - table = pq.read_table(path, memory_map=True).combine_chunks() + table = pq.read_table(path, memory_map=False).combine_chunks() for idx, batch in enumerate(table.to_batches(max_chunksize=2_500_000)): batch: RecordBatch