Merge pull request #1389 from ashitsalesforce/master

do not re-read from the same CSV or database during an operation if c…
forcedotcom · Nov 28, 2024 · 1fb10bc · 1fb10bc
2 parents 9e07e27 + 701780c
commit 1fb10bc
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 15 deletions.
diff --git a/src/main/java/com/salesforce/dataloader/dao/DAORowCache.java b/src/main/java/com/salesforce/dataloader/dao/DAORowCache.java
@@ -33,7 +33,7 @@
 public class DAORowCache {
     private ArrayList<TableRow> rowList = new ArrayList<TableRow>();
     private int currentRowIndex = 0;
-    private int totalRows = 0;
+    private int cachedRows = 0;
 
     public DAORowCache() {
     }
@@ -44,16 +44,23 @@ public void resetCurrentRowIndex() {
 
     public TableRow getCurrentRow() {
         AppConfig appConfig = AppConfig.getCurrentConfig();
-        if (currentRowIndex >= totalRows
+        if (currentRowIndex >= cachedRows
             || !appConfig.getBoolean(AppConfig.PROP_PROCESS_BULK_CACHE_DATA_FROM_DAO)) {
             return null;
         }
         return rowList.get(currentRowIndex++);
     }
 
     public void addRow(TableRow row) {
-        rowList.add(row);
+        // add a row to the cache only if it is not cached already
+        if (currentRowIndex >= cachedRows) {
+            rowList.add(row);
+            cachedRows++;
+        }
         currentRowIndex++;
-        totalRows++;
+    }
+
+    public int getCachedRows() {
+        return cachedRows;
     }
 }
diff --git a/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java b/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java
@@ -127,11 +127,13 @@ public void open() throws DataAccessObjectInitializationException {
         if (isOpen) {
             close();
         }
+        if (!appConfig.getBoolean(AppConfig.PROP_PROCESS_BULK_CACHE_DATA_FROM_DAO)
+                || rowCache.getCachedRows() == 0) {
+            initalizeInput(csvDelimiters);
+            readHeaderRow();
+        }
         currentRowNumber = 0;
         rowCache.resetCurrentRowIndex();
-
-        initalizeInput(csvDelimiters);
-        readHeaderRow();
         isOpen = true;
     }
 

diff --git a/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java b/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java
@@ -115,15 +115,22 @@ public void open() throws DataAccessObjectInitializationException {
      * @param params
      * @throws DataAccessObjectInitializationException
      */
-    public void open(Map<String,Object> params) throws DataAccessObjectInitializationException {
-        currentRowNumber = 0;
-        try {
-            setupQuery(params);
-        } catch (DataAccessObjectInitializationException e) {
-            throw e;
-        } catch (Exception e) {
-            throw new DataAccessObjectInitializationException(e.getMessage(), e);
+    private void open(Map<String,Object> params) throws DataAccessObjectInitializationException {
+        if (dbContext.isOpen()) {
+            close();
         }
+        if (!appConfig.getBoolean(AppConfig.PROP_PROCESS_BULK_CACHE_DATA_FROM_DAO)
+                || rowCache.getCachedRows() == 0) {
+            try {
+                setupQuery(params);
+            } catch (DataAccessObjectInitializationException e) {
+                throw e;
+            } catch (Exception e) {
+                throw new DataAccessObjectInitializationException(e.getMessage(), e);
+            }
+        }
+        currentRowNumber = 0;
+        rowCache.resetCurrentRowIndex();
         dbContext.setOpen(true);
     }