Skip to content

Commit

Permalink
Buffer correctly
Browse files Browse the repository at this point in the history
  • Loading branch information
orf committed Oct 20, 2024
1 parent b4ff0a4 commit 6d1e39d
Showing 1 changed file with 8 additions and 5 deletions.
13 changes: 8 additions & 5 deletions src/pypi_data/cli.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import asyncio
import contextlib
import gzip
import io
from collections import defaultdict
from pathlib import Path
from typing import (
Expand Down Expand Up @@ -56,11 +57,13 @@ def github_client(github_token) -> Github:
@contextlib.contextmanager
def open_path(path: Path, mode: Literal["wb", "rb"]) -> Generator[BinaryIO, None, None]:
buffer_size = 1024 * 1024 * 50
with path.open(mode, buffering=buffer_size) as fd:
if path.suffix == ".gz":
with gzip.open(fd, mode) as gzip_fd:
yield gzip_fd
else:

if path.suffix == ".gz":
with gzip.open(path, mode) as gzip_fd:
with io.BufferedWriter(gzip_fd, buffer_size=buffer_size) as buffered_fd:
yield buffered_fd
else:
with path.open(mode, buffering=buffer_size) as fd:
yield fd

log.info(
Expand Down

0 comments on commit 6d1e39d

Please sign in to comment.