Skip to content

Commit

Permalink
Remove limit
Browse files Browse the repository at this point in the history
  • Loading branch information
orf committed Oct 20, 2024
1 parent caadb58 commit eeeb293
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions src/pypi_data/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,14 +158,14 @@ def merge_datasets(
repo_path: Path,
output: Path,
max_buffer_size: Annotated[str, typer.Option()] = "5GB",
target_size: Annotated[str, typer.Option()] = "1.8GB",
target_size: Annotated[str, typer.Option()] = "1.9GB",
):
with open_path(repo_path, mode="rb") as fd:
repos = Repos.model_validate_json(fd.read()).root
max_buffer_size = pydantic.RootModel[ByteSize].model_validate(max_buffer_size).root
target_size = pydantic.RootModel[ByteSize].model_validate(target_size).root
# Debug failures...
asyncio.run(combine_parquet(repos[200:], output, max_buffer_size, target_size))
asyncio.run(combine_parquet(repos, output, max_buffer_size, target_size))


async def resolve_dataset_redirects(
Expand Down

0 comments on commit eeeb293

Please sign in to comment.