Fix pl-court-raw dataset by filtering records with empty content from

pwr-ai · Aug 27, 2024 · 5afd7e3 · 5afd7e3
1 parent 7d40fc5
commit 5afd7e3
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 6 deletions.
diff --git a/data/datasets/pl/raw.dvc b/data/datasets/pl/raw.dvc
@@ -1,6 +1,6 @@
 outs:
-- md5: 5dd44be2eea852bcce3d0918ff8b97da.dir
-  size: 10234880729
+- md5: 622ba21868561c26fb6877ad95bfb5c5.dir
+  size: 10234505621
   nfiles: 17
   hash: md5
   path: raw
diff --git a/scripts/README.md b/scripts/README.md
@@ -52,8 +52,9 @@ MONGO_DB_NAME="datasets"
    to remote storage:
     ```shell
     PYTHONPATH=.  python scripts/dataset/dump_pl_dataset.py \
-        --file-name data/datasets/pl/raw/raw.parquet
-    dvc add data/datasets/pl/raw/raw.parquet && dvc push
+        --file-name data/datasets/pl/raw/raw.parquet \
+        --filter-empty-content
+    dvc add data/datasets/pl/raw && dvc push
     ```
 7. Generate dataset card for `pl-court-raw`
     ```shell

diff --git a/scripts/dataset/dump_pl_dataset.py b/scripts/dataset/dump_pl_dataset.py
@@ -29,12 +29,11 @@ def main(
     collection = get_mongo_collection(mongo_uri=mongo_uri)
 
     if filter_empty_content:
-        query = {"content": {"$ne": True}}
+        query = {"content": {"$ne": None}}
     else:
         query = {}
 
     num_docs = collection.count_documents(query)
-
     dumped_data = list(file_name.parent.glob("*.parquet"))
     start_offset = 0
     if dumped_data: