Remove lto

.github/workflows/unique_python_files.yml

@@ -22,6 +22,9 @@ jobs:
           toolchain: stable
           override: true
 
+      - name: Install parquet-cli
+        run: cargo install parquet --features=cli
+
       - uses: Swatinem/rust-cache@v2
         with:
           prefix-key: unique-python-files-
@@ -37,3 +40,7 @@ jobs:
       - name: Run
         run: |
           ./target/optimized/data links/dataset.txt data/
+
+      - name: Combine
+        run: |
+          parquet-concat data/combined.parquet data/output/**/*.parquet

Cargo.toml

@@ -19,5 +19,5 @@ snmalloc-rs = "0.3"
 
 [profile.optimized]
 inherits = "release"
-lto = true
-codegen-units = 1
+#lto = true
+#codegen-units = 1

sql/unique_files.sql

@@ -6,3 +6,4 @@ FROM (SELECT hash,
       FROM input_dataset
       where skip_reason = ''
         and archive_path ILIKE '%.py' and size != 0) as ordered;
+WHERE n = 1

sql/unique_files_combined.sql

@@ -0,0 +1,8 @@
+SELECT hash, repository, uploaded_on
+FROM (SELECT hash,
+             repository,
+             uploaded_on,
+             ROW_NUMBER() OVER (PARTITION BY encode(hash, 'hex') order by uploaded_on) as n
+      FROM input_dataset
+      ) as ordered;
+WHERE n = 1

src/main.rs

@@ -46,9 +46,10 @@ async fn main() -> Result<()> {
         let path = download_dir.join(format!("url-{}.parquet", idx));
         let output_dir = output_dir.join(format!("url-{}/", idx));
         download_file(&url, &path).await?;
-        get_unique_python_files(&path, &output_dir).await?;
+        get_unique_python_files(&path, &output_dir, include_str!("../sql/unique_files.sql")).await?;
         tokio::fs::remove_file(&path).await?;
     }
+    get_unique_python_files(Path::new("data/combined.parquet"), Path::new("data/combined/"), include_str!("../sql/unique_files_combined.sql")).await?;
     Ok(())
 }
 
@@ -82,12 +83,12 @@ async fn download_file(url: &str, path: &Path) -> Result<()> {
     Ok(())
 }
 
-async fn get_unique_python_files(path: &Path, output: &Path) -> Result<()> {
+async fn get_unique_python_files(path: &Path, output: &Path, sql: &str) -> Result<()> {
     let ctx = SessionContext::new();
     let read_options = ParquetReadOptions::default().parquet_pruning(true);
     ctx.register_parquet("input_dataset", path.to_str().unwrap(), read_options).await?;
 
-    let df = ctx.sql(include_str!("../sql/unique_files.sql")).await.unwrap();
+    let df = ctx.sql(sql).await.unwrap();
 
     let props = WriterProperties::builder()
         .set_compression(Compression::ZSTD(ZstdLevel::try_new(13).unwrap()))
@@ -99,4 +100,4 @@ async fn get_unique_python_files(path: &Path, output: &Path) -> Result<()> {
         Some(props),
     ).await?;
     Ok(())
-}
+}