forcedotcom · ashitsalesforce · Nov 30, 2024 · Nov 30, 2024
diff --git a/src/main/java/com/salesforce/dataloader/dao/AbstractDataReaderImpl.java b/src/main/java/com/salesforce/dataloader/dao/AbstractDataReaderImpl.java
@@ -46,7 +46,7 @@ public AbstractDataReaderImpl(AppConfig appConfig) {
 
     public List<TableRow> readTableRowList(int maxRows) throws DataAccessObjectException {
         List<TableRow> rowList = null;
-        if (this.rowCache.size() >= this.currentRowNumber + maxRows) {
+        if (this.rowCache.size() > this.currentRowNumber + maxRows) {
             rowList = this.rowCache.getRows(currentRowNumber, maxRows);
             currentRowNumber = currentRowNumber + rowList.size();
             return rowList;

diff --git a/src/main/java/com/salesforce/dataloader/dao/DataReader.java b/src/main/java/com/salesforce/dataloader/dao/DataReader.java
@@ -56,15 +56,6 @@ public interface DataReader extends DataAccessObject {
      */
     TableRow readTableRow() throws DataAccessObjectException;
 
-    /**
-     * Get a list of rows of data from a data source
-     *
-     * @param maxRows Maximum number of rows to read in one call
-     * @return a list of up to maxRows {@link Row} objects, each of them containing all the keys and values of a row
-     * @throws DataAccessObjectException
-     */
-    List<Row> readRowList(int maxRows) throws DataAccessObjectException;
-
     /**
      * Get a list of rows of data from a data source
      *

diff --git a/src/main/java/com/salesforce/dataloader/dao/DataWriter.java b/src/main/java/com/salesforce/dataloader/dao/DataWriter.java
@@ -31,6 +31,7 @@
 import com.salesforce.dataloader.exception.DataAccessObjectException;
 import com.salesforce.dataloader.exception.DataAccessObjectInitializationException;
 import com.salesforce.dataloader.model.Row;
+import com.salesforce.dataloader.model.TableRow;
 
 /**
  * Interface to be implemented for data writers -- data access objects that are used for writing rows of data.
@@ -63,4 +64,6 @@ public interface DataWriter extends DataAccessObject {
     boolean writeRowList(List<Row> inputRowList) throws DataAccessObjectException;
 
     public List<String> getColumnNamesFromRow(Row row) throws DataAccessObjectInitializationException;
+
+    public boolean writeTableRowList(List<TableRow> rows) throws DataAccessObjectException;
 }
diff --git a/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java b/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java
@@ -145,22 +145,6 @@ protected boolean isOpenFlag() {
         return this.isOpen;
     }
 
-    @Override
-    public List<Row> readRowList(int maxRows) throws DataAccessObjectException {
-        List<Row> outputRows = new ArrayList<Row>();
-        for (int i = 0; i < maxRows; i++) {
-            Row outputRow = readRow();
-            if (outputRow != null) {
-                // if row has been returned, add it to the output
-                outputRows.add(outputRow);
-            } else {
-                // if encountered null, the reading is over
-                break;
-            }
-        }
-        return outputRows;
-    }
-
     @Override
     protected List<TableRow> readTableRowListFromDAO(int maxRows) throws DataAccessObjectException {
         List<TableRow> outputRows = new ArrayList<TableRow>();

diff --git a/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileWriter.java b/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileWriter.java
@@ -44,6 +44,7 @@
 import com.salesforce.dataloader.exception.DataAccessObjectException;
 import com.salesforce.dataloader.exception.DataAccessObjectInitializationException;
 import com.salesforce.dataloader.model.Row;
+import com.salesforce.dataloader.model.TableRow;
 import com.salesforce.dataloader.util.AppUtil;
 
 /**
@@ -211,7 +212,21 @@ public boolean writeRowList(List<Row> rows) throws DataAccessObjectException {
         }
         return success;
     }
-
+
+    /*
+     * (non-Javadoc)
+     * @see com.salesforce.dataloader.dao.csv.Writer#writeRowList(java.util.List)
+     */
+    @Override
+    public boolean writeTableRowList(List<TableRow> rows) throws DataAccessObjectException {
+        boolean success = true;
+        // return the last result, should be same as others
+        for (TableRow trow : rows) {
+            success = writeRow(trow.convertToRow());
+        }
+        return success;
+    }
+
     private void visitHeaderColumns(List<String> columnNames, CSVColumnVisitor visitor) throws IOException {
         for (String colName : columnNames) {
             String outColName;

diff --git a/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java b/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java
@@ -160,30 +160,10 @@ private void setupQuery(Map<String,Object> params) throws DataAccessObjectInitia
             throw new DataAccessObjectInitializationException(errMsg, sqe);
         }
     }
-
-    /*
-     * (non-Javadoc)
-     * @see com.salesforce.dataloader.dao.DataReader#readRowList(int)
-     */
-    @Override
-    public List<Row> readRowList(int maxRows) throws DataAccessObjectException {
-        List<Row> outputRows = new ArrayList<Row>();
-        for(int i=0; i < maxRows; i++) {
-            Row outputRow = readRow();
-            if(outputRow != null) {
-                // if row has been returned, add it to the output
-                outputRows.add(outputRow);
-            } else {
-                // if encountered null, the reading is over
-                break;
-            }
-        }
-        return outputRows;
-    }
 
     /*
      * (non-Javadoc)
-     * @see com.salesforce.dataloader.dao.DataReader#readRowList(int)
+     * @see com.salesforce.dataloader.dao.DataReader#readTableRowList(int)
      */
     @Override
     protected List<TableRow> readTableRowListFromDAO(int maxRows) throws DataAccessObjectException {

diff --git a/src/main/java/com/salesforce/dataloader/dao/database/DatabaseWriter.java b/src/main/java/com/salesforce/dataloader/dao/database/DatabaseWriter.java
@@ -30,6 +30,7 @@
 import java.util.*;
 
 import com.salesforce.dataloader.model.Row;
+import com.salesforce.dataloader.model.TableRow;
 import org.apache.commons.dbcp2.BasicDataSource;
 import org.apache.logging.log4j.Logger;
 import com.salesforce.dataloader.util.DLLogManager;
@@ -119,7 +120,7 @@ private void setupUpdate() throws DataAccessObjectInitializationException {
         dbContext.replaceSqlParams(sqlConfig.getSqlString());
         dbContext.prepareStatement();
     }
-
+    
     /*
      * (non-Javadoc)
      * @see com.salesforce.dataloader.dao.DataWriter#writeRowList(java.util.List)
@@ -285,4 +286,16 @@ public void setColumnNames(List<String> columnNames) {
     public List<String> getColumnNamesFromRow(Row row) throws DataAccessObjectInitializationException {
         return null;
     }
+
+    @Override
+    public boolean writeTableRowList(List<TableRow> trowList) throws DataAccessObjectException {
+        if (trowList == null) {
+            return false;
+        }
+        ArrayList<Row> rowList = new ArrayList<Row>(trowList.size());
+        for (TableRow trow : trowList) {
+            rowList.add(trow.convertToRow());
+        }
+        return writeRowList(rowList);
+    }
 }
diff --git a/src/main/java/com/salesforce/dataloader/model/TableRow.java b/src/main/java/com/salesforce/dataloader/model/TableRow.java
@@ -26,7 +26,7 @@
 package com.salesforce.dataloader.model;
 
 import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Arrays;
 
 public class TableRow {
     private TableHeader header;
@@ -36,6 +36,11 @@ public TableRow(TableHeader header) {
         this.header = header;
         cellValues = new Object[header.getColumns().size()];
     }
+
+    public TableRow(TableRow rowToCopy) {
+        this.header = rowToCopy.getHeader();
+        cellValues = Arrays.copyOf(rowToCopy.cellValues, rowToCopy.cellValues.length);
+    }
 
     public Object get(String key) {
         Integer colPos = this.header.getColumnPosition(key);
@@ -77,4 +82,14 @@ public static TableRow singleEntryImmutableRow(String key, Object value) {
         row.put(key, value);
         return row;
     }
+
+    public int getNonEmptyCellsCount() {
+        int numNonEmptyCells = 0;
+        for (int i = 0; i < this.cellValues.length; i++) {
+            if (cellValues[i] != null) {
+                numNonEmptyCells++;
+            }
+        }
+        return numNonEmptyCells;
+    }
 }
diff --git a/src/test/java/com/salesforce/dataloader/dao/database/DatabaseTest.java b/src/test/java/com/salesforce/dataloader/dao/database/DatabaseTest.java
@@ -29,6 +29,7 @@
 import com.salesforce.dataloader.controller.Controller;
 import com.salesforce.dataloader.exception.DataAccessObjectException;
 import com.salesforce.dataloader.model.Row;
+import com.salesforce.dataloader.model.TableRow;
 import com.salesforce.dataloader.util.AccountRowComparator;
 import org.apache.logging.log4j.Logger;
 import com.salesforce.dataloader.util.DLLogManager;
@@ -140,7 +141,7 @@ private static void verifyDbInsertOrUpdate(Controller theController, boolean isI
             reader = new DatabaseReader(theController.getAppConfig(), "queryAccountAll");
             reader.open();
             int readBatchSize = 1000;
-            List<Row> readRowList = reader.readRowList(readBatchSize);
+            List<TableRow> readRowList = reader.readTableRowList(readBatchSize);
             int rowsProcessed = 0;
             assertNotNull("Error reading " + readBatchSize + " rows", readRowList);
             while(readRowList.size() > 0) {
@@ -149,18 +150,18 @@ private static void verifyDbInsertOrUpdate(Controller theController, boolean isI
                 Collections.sort(readRowList, new AccountRowComparator(!isInsert));
                 logger.info("Verifying database success for next " + (rowsProcessed + readRowList.size()) + " rows");
                 for (int i=0; i < readRowList.size(); i++) {
-                    Row readRow = readRowList.get(i);
+                    TableRow readRow = readRowList.get(i);
                     assertNotNull("Error reading data row #" + i + ": the row shouldn't be null", readRow);
-                    assertTrue("Error reading data row #" + i + ": the row shouldn't be empty", readRow.size() > 0);
+                    assertTrue("Error reading data row #" + i + ": the row shouldn't be empty", readRow.getNonEmptyCellsCount() > 0);
                     Row expectedRow = DatabaseTestUtil.getInsertOrUpdateAccountRow(isInsert, rowsProcessed, DatabaseTestUtil.DateType.VALIDATION);
                     // verify all expected data
                     for (String colName : VALIDATE_COLS) {
-                        verifyCol(colName, readRow, expectedRow);
+                        verifyCol(colName, readRow.convertToRow(), expectedRow);
                     }
 
                     rowsProcessed++;
                 }
-                readRowList = reader.readRowList(readBatchSize);
+                readRowList = reader.readTableRowList(readBatchSize);
                 assertNotNull("Error reading " + readBatchSize + " rows", readRowList);
             }
         } finally {

diff --git a/src/test/java/com/salesforce/dataloader/process/CsvExtractProcessTest.java b/src/test/java/com/salesforce/dataloader/process/CsvExtractProcessTest.java
@@ -35,6 +35,7 @@
 import com.salesforce.dataloader.exception.DataAccessObjectException;
 import com.salesforce.dataloader.exception.ProcessInitializationException;
 import com.salesforce.dataloader.model.Row;
+import com.salesforce.dataloader.model.TableRow;
 import com.sforce.soap.partner.GetUserInfoResult;
 import com.sforce.soap.partner.SaveResult;
 import com.sforce.soap.partner.sobject.SObject;
@@ -322,7 +323,7 @@ private void testBinaryDataInRTFQueryResult(String inclueRTFBinaryData) throws E
         final CSVFileReader successRdr = new CSVFileReader(new File(fileName), getController().getAppConfig(), true, false);
         String idFieldName = this.isBulkV2APIEnabled(insertArgMap)?"sf__Id":"ID";
         try {
-            for (Row row : successRdr.readRowList(Integer.MAX_VALUE)) {
+            for (TableRow row : successRdr.readTableRowList(Integer.MAX_VALUE)) {
                 final String rowId = (String) row.get(idFieldName);
                 if (rowId != null) {
                     ids.add(rowId);

diff --git a/src/test/java/com/salesforce/dataloader/process/CsvHardDeleteTest.java b/src/test/java/com/salesforce/dataloader/process/CsvHardDeleteTest.java
@@ -33,6 +33,8 @@
 import com.salesforce.dataloader.exception.DataAccessObjectException;
 import com.salesforce.dataloader.exception.DataAccessObjectInitializationException;
 import com.salesforce.dataloader.model.Row;
+import com.salesforce.dataloader.model.TableRow;
+
 import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -172,7 +174,7 @@ public InvalidIdTemplateListener(int numValidAccounts) {
         }
 
         @Override
-        public void updateRow(int idx, Row row) {
+        public void updateRow(int idx, TableRow row) {
             if (idx == 0)
                 row.put("ID", "abcde0123456789XYZ");
             else
@@ -237,7 +239,7 @@ public HeterogeneousIdTemplateListener(int numAccounts, int numContacts) {
         }
 
         @Override
-        public void updateRow(int idx, Row row) {
+        public void updateRow(int idx, TableRow row) {
             if (idx < this.contactIds.length)
                 row.put("ID", this.contactIds[idx]);
             else

diff --git a/src/test/java/com/salesforce/dataloader/process/CsvProcessAttachmentTest.java b/src/test/java/com/salesforce/dataloader/process/CsvProcessAttachmentTest.java
@@ -36,6 +36,7 @@
 import com.salesforce.dataloader.exception.ProcessInitializationException;
 import com.salesforce.dataloader.exception.UnsupportedOperationException;
 import com.salesforce.dataloader.model.Row;
+import com.salesforce.dataloader.model.TableRow;
 import com.sforce.soap.partner.QueryResult;
 import com.sforce.soap.partner.SaveResult;
 import com.sforce.soap.partner.sobject.SObject;
@@ -181,7 +182,7 @@ public AttachmentTemplateListener() {
         }
 
         @Override
-        public void updateRow(int idx, Row row) {
+        public void updateRow(int idx, TableRow row) {
             // set parent account id
             row.put("ParentId", getAccountIds()[0]);
             // make body pathname absolute

diff --git a/src/test/java/com/salesforce/dataloader/process/CsvProcessTest.java b/src/test/java/com/salesforce/dataloader/process/CsvProcessTest.java
@@ -51,6 +51,7 @@
 import com.salesforce.dataloader.exception.DataAccessObjectException;
 import com.salesforce.dataloader.exception.ProcessInitializationException;
 import com.salesforce.dataloader.model.Row;
+import com.salesforce.dataloader.model.TableRow;
 import com.salesforce.dataloader.util.AppUtil;
 import com.sforce.soap.partner.sobject.SObject;
 
@@ -367,7 +368,7 @@ private SObject[] retrieveAccounts(Controller resultController, String... accoun
         final CSVFileReader successRdr = new CSVFileReader(new File(fileName), getController().getAppConfig(), true, false);
         try {
             // TODO: revise the use of Integer.MAX_VALUE
-            for (Row row : successRdr.readRowList(Integer.MAX_VALUE)) {
+            for (TableRow row : successRdr.readTableRowList(Integer.MAX_VALUE)) {
                 final String rowId = (String) row.get("ID");
                 if (rowId != null) {
                     ids.add(rowId);
@@ -446,7 +447,7 @@ public AttachmentTemplateListener() {
         }
 
         @Override
-        public void updateRow(int idx, Row row) {
+        public void updateRow(int idx, TableRow row) {
             // set parent account id
             row.put("ParentId", getAccountIds()[0]);
             // make body pathname absolute

diff --git a/src/test/java/com/salesforce/dataloader/process/DatabaseProcessTest.java b/src/test/java/com/salesforce/dataloader/process/DatabaseProcessTest.java
@@ -47,7 +47,7 @@
 import com.salesforce.dataloader.dao.database.DatabaseReader;
 import com.salesforce.dataloader.dao.database.DatabaseTestUtil;
 import com.salesforce.dataloader.exception.*;
-import com.salesforce.dataloader.model.Row;
+import com.salesforce.dataloader.model.TableRow;
 
 /**
  * Automated tests for dataloader database batch interface
@@ -204,14 +204,14 @@ private void verifyDbSuccess(Controller theController, String dbConfigName, int
             reader = new DatabaseReader(theController.getAppConfig(), dbConfigName);
             reader.open();
             int readBatchSize = theController.getAppConfig().getInt(AppConfig.PROP_DAO_READ_BATCH_SIZE);
-            List<Row> successRows = reader.readRowList(readBatchSize);
+            List<TableRow> successRows = reader.readTableRowList(readBatchSize);
             int rowsProcessed = 0;
             assertNotNull("Error reading " + readBatchSize + " rows", successRows);
             while(successRows.size() > 0) {
                 rowsProcessed += successRows.size();
                 logger.info("Verifying database success for next " + successRows.size() + " of total " + rowsProcessed + " rows");
                 assertTrue("No updated rows have been found in the database.", successRows.size() > 0);
-                successRows = reader.readRowList(readBatchSize);
+                successRows = reader.readTableRowList(readBatchSize);
             }
             assertEquals(expectedSuccesses, rowsProcessed);
         } catch (DataAccessObjectInitializationException e) {