feat: replicate crates data on bigquery (#2589)

* add: `archive2bq` factory asset * feat: replicate `crates` data on bigquery
opensource-observer · Dec 5, 2024 · a8f3b0e · a8f3b0e
1 parent 287561c
commit a8f3b0e
Show file tree

Hide file tree

Showing 2 changed files with 427 additions and 0 deletions.
diff --git a/warehouse/oso_dagster/assets/crates.py b/warehouse/oso_dagster/assets/crates.py
@@ -0,0 +1,21 @@
+from google.cloud.bigquery import SourceFormat
+from ..constants import staging_bucket
+from ..factories.archive2bq import Archive2BqAssetConfig, create_archive2bq_asset
+
+crates_data = create_archive2bq_asset(
+    Archive2BqAssetConfig(
+        key_prefix="rust",
+        asset_name="crates",
+        source_url="https://static.crates.io/db-dump.tar.gz",
+        source_format=SourceFormat.CSV,
+        filter_fn=lambda file: file.endswith(".csv"),
+        schema_overrides={
+            "crates": {
+                "id": "INTEGER",
+            }
+        },
+        staging_bucket=staging_bucket,
+        dataset_id="crates",
+        deps=[],
+    )
+)