Spark 3.2: Update task stats for split files (#4446)

Co-authored-by: Prashant Singh <[email protected]>
linkedin · Apr 3, 2022 · 4ab4b91 · 4ab4b91
1 parent 6f4ba69
commit 4ab4b91
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java
@@ -148,7 +148,9 @@ protected Statistics estimateStatistics(Snapshot snapshot) {
 
     for (CombinedScanTask task : tasks()) {
       for (FileScanTask file : task.files()) {
-        numRows += file.file().recordCount();
+        // TODO: if possible, take deletes also into consideration.
+        double fractionOfFileScanned = ((double) file.length()) / file.file().fileSizeInBytes();
+        numRows += (fractionOfFileScanned * file.file().recordCount());
       }
     }