Skip to content

Commit

Permalink
Add pydantic support to rl.utils.io.write_jsonl
Browse files Browse the repository at this point in the history
  • Loading branch information
ProbablyFaiz committed Sep 30, 2024
1 parent 4962d9b commit 58b6d98
Showing 1 changed file with 7 additions and 1 deletion.
8 changes: 7 additions & 1 deletion rl/utils/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import dotenv
import requests
import tqdm
from pydantic import BaseModel

DOTENV_LOADED = False

Expand Down Expand Up @@ -98,7 +99,12 @@ def write_jsonl(filename: str | Path, records: Iterable[Any], overwrite=False) -
raise ValueError(f"{filename} already exists and overwrite is not set.")
with filename.open("w") as f:
for record in records:
f.write(json.dumps(record) + "\n")
json_record: str
if isinstance(record, BaseModel):
json_record = record.model_dump_json()
else:
json_record = json.dumps(record)
f.write(json_record + "\n")


def write_jsonl_spark(filename: str | Path, df, overwrite=False) -> None:
Expand Down

0 comments on commit 58b6d98

Please sign in to comment.