diff --git a/src/main/java/com/salesforce/dataloader/dao/AbstractDataReaderImpl.java b/src/main/java/com/salesforce/dataloader/dao/AbstractDataReaderImpl.java index 4fe1e06a..534272d2 100644 --- a/src/main/java/com/salesforce/dataloader/dao/AbstractDataReaderImpl.java +++ b/src/main/java/com/salesforce/dataloader/dao/AbstractDataReaderImpl.java @@ -46,7 +46,7 @@ public AbstractDataReaderImpl(AppConfig appConfig) { public List<TableRow> readTableRowList(int maxRows) throws DataAccessObjectException { List<TableRow> rowList = null; - if (this.rowCache.size() >= this.currentRowNumber + maxRows) { + if (this.rowCache.size() > this.currentRowNumber + maxRows) { rowList = this.rowCache.getRows(currentRowNumber, maxRows); currentRowNumber = currentRowNumber + rowList.size(); return rowList; diff --git a/src/main/java/com/salesforce/dataloader/dao/DataReader.java b/src/main/java/com/salesforce/dataloader/dao/DataReader.java index 06514767..8a48ba42 100644 --- a/src/main/java/com/salesforce/dataloader/dao/DataReader.java +++ b/src/main/java/com/salesforce/dataloader/dao/DataReader.java @@ -56,15 +56,6 @@ public interface DataReader extends DataAccessObject { */ TableRow readTableRow() throws DataAccessObjectException; - /** - * Get a list of rows of data from a data source - * - * @param maxRows Maximum number of rows to read in one call - * @return a list of up to maxRows {@link Row} objects, each of them containing all the keys and values of a row - * @throws DataAccessObjectException - */ - List<Row> readRowList(int maxRows) throws DataAccessObjectException; - /** * Get a list of rows of data from a data source * diff --git a/src/main/java/com/salesforce/dataloader/dao/DataWriter.java b/src/main/java/com/salesforce/dataloader/dao/DataWriter.java index 1de53d92..f39a1173 100644 --- a/src/main/java/com/salesforce/dataloader/dao/DataWriter.java +++ b/src/main/java/com/salesforce/dataloader/dao/DataWriter.java @@ -31,6 +31,7 @@ import com.salesforce.dataloader.exception.DataAccessObjectException; import com.salesforce.dataloader.exception.DataAccessObjectInitializationException; import com.salesforce.dataloader.model.Row; +import com.salesforce.dataloader.model.TableRow; /** * Interface to be implemented for data writers -- data access objects that are used for writing rows of data. @@ -63,4 +64,6 @@ public interface DataWriter extends DataAccessObject { boolean writeRowList(List<Row> inputRowList) throws DataAccessObjectException; public List<String> getColumnNamesFromRow(Row row) throws DataAccessObjectInitializationException; + + public boolean writeTableRowList(List<TableRow> rows) throws DataAccessObjectException; } diff --git a/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java b/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java index c7066fd9..7824aec1 100644 --- a/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java +++ b/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java @@ -145,22 +145,6 @@ protected boolean isOpenFlag() { return this.isOpen; } - @Override - public List<Row> readRowList(int maxRows) throws DataAccessObjectException { - List<Row> outputRows = new ArrayList<Row>(); - for (int i = 0; i < maxRows; i++) { - Row outputRow = readRow(); - if (outputRow != null) { - // if row has been returned, add it to the output - outputRows.add(outputRow); - } else { - // if encountered null, the reading is over - break; - } - } - return outputRows; - } - @Override protected List<TableRow> readTableRowListFromDAO(int maxRows) throws DataAccessObjectException { List<TableRow> outputRows = new ArrayList<TableRow>(); diff --git a/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileWriter.java b/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileWriter.java index 8eacb36b..4b1d4bdd 100644 --- a/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileWriter.java +++ b/src/main/java/com/salesforce/dataloader/dao/csv/CSVFileWriter.java @@ -44,6 +44,7 @@ import com.salesforce.dataloader.exception.DataAccessObjectException; import com.salesforce.dataloader.exception.DataAccessObjectInitializationException; import com.salesforce.dataloader.model.Row; +import com.salesforce.dataloader.model.TableRow; import com.salesforce.dataloader.util.AppUtil; /** @@ -211,7 +212,21 @@ public boolean writeRowList(List<Row> rows) throws DataAccessObjectException { } return success; } - + + /* + * (non-Javadoc) + * @see com.salesforce.dataloader.dao.csv.Writer#writeRowList(java.util.List) + */ + @Override + public boolean writeTableRowList(List<TableRow> rows) throws DataAccessObjectException { + boolean success = true; + // return the last result, should be same as others + for (TableRow trow : rows) { + success = writeRow(trow.convertToRow()); + } + return success; + } + private void visitHeaderColumns(List<String> columnNames, CSVColumnVisitor visitor) throws IOException { for (String colName : columnNames) { String outColName; diff --git a/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java b/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java index 5cac2b5a..80804aa9 100644 --- a/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java +++ b/src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java @@ -160,30 +160,10 @@ private void setupQuery(Map<String,Object> params) throws DataAccessObjectInitia throw new DataAccessObjectInitializationException(errMsg, sqe); } } - - /* - * (non-Javadoc) - * @see com.salesforce.dataloader.dao.DataReader#readRowList(int) - */ - @Override - public List<Row> readRowList(int maxRows) throws DataAccessObjectException { - List<Row> outputRows = new ArrayList<Row>(); - for(int i=0; i < maxRows; i++) { - Row outputRow = readRow(); - if(outputRow != null) { - // if row has been returned, add it to the output - outputRows.add(outputRow); - } else { - // if encountered null, the reading is over - break; - } - } - return outputRows; - } /* * (non-Javadoc) - * @see com.salesforce.dataloader.dao.DataReader#readRowList(int) + * @see com.salesforce.dataloader.dao.DataReader#readTableRowList(int) */ @Override protected List<TableRow> readTableRowListFromDAO(int maxRows) throws DataAccessObjectException { diff --git a/src/main/java/com/salesforce/dataloader/dao/database/DatabaseWriter.java b/src/main/java/com/salesforce/dataloader/dao/database/DatabaseWriter.java index 10769376..f90d6ab7 100644 --- a/src/main/java/com/salesforce/dataloader/dao/database/DatabaseWriter.java +++ b/src/main/java/com/salesforce/dataloader/dao/database/DatabaseWriter.java @@ -30,6 +30,7 @@ import java.util.*; import com.salesforce.dataloader.model.Row; +import com.salesforce.dataloader.model.TableRow; import org.apache.commons.dbcp2.BasicDataSource; import org.apache.logging.log4j.Logger; import com.salesforce.dataloader.util.DLLogManager; @@ -119,7 +120,7 @@ private void setupUpdate() throws DataAccessObjectInitializationException { dbContext.replaceSqlParams(sqlConfig.getSqlString()); dbContext.prepareStatement(); } - + /* * (non-Javadoc) * @see com.salesforce.dataloader.dao.DataWriter#writeRowList(java.util.List) @@ -285,4 +286,16 @@ public void setColumnNames(List<String> columnNames) { public List<String> getColumnNamesFromRow(Row row) throws DataAccessObjectInitializationException { return null; } + + @Override + public boolean writeTableRowList(List<TableRow> trowList) throws DataAccessObjectException { + if (trowList == null) { + return false; + } + ArrayList<Row> rowList = new ArrayList<Row>(trowList.size()); + for (TableRow trow : trowList) { + rowList.add(trow.convertToRow()); + } + return writeRowList(rowList); + } } diff --git a/src/main/java/com/salesforce/dataloader/model/TableRow.java b/src/main/java/com/salesforce/dataloader/model/TableRow.java index 3e7d5b4f..ef14e6f5 100644 --- a/src/main/java/com/salesforce/dataloader/model/TableRow.java +++ b/src/main/java/com/salesforce/dataloader/model/TableRow.java @@ -26,7 +26,7 @@ package com.salesforce.dataloader.model; import java.util.ArrayList; -import java.util.Collections; +import java.util.Arrays; public class TableRow { private TableHeader header; @@ -36,6 +36,11 @@ public TableRow(TableHeader header) { this.header = header; cellValues = new Object[header.getColumns().size()]; } + + public TableRow(TableRow rowToCopy) { + this.header = rowToCopy.getHeader(); + cellValues = Arrays.copyOf(rowToCopy.cellValues, rowToCopy.cellValues.length); + } public Object get(String key) { Integer colPos = this.header.getColumnPosition(key); @@ -77,4 +82,14 @@ public static TableRow singleEntryImmutableRow(String key, Object value) { row.put(key, value); return row; } + + public int getNonEmptyCellsCount() { + int numNonEmptyCells = 0; + for (int i = 0; i < this.cellValues.length; i++) { + if (cellValues[i] != null) { + numNonEmptyCells++; + } + } + return numNonEmptyCells; + } } \ No newline at end of file diff --git a/src/test/java/com/salesforce/dataloader/dao/database/DatabaseTest.java b/src/test/java/com/salesforce/dataloader/dao/database/DatabaseTest.java index 5ccf91b6..cd3601bb 100644 --- a/src/test/java/com/salesforce/dataloader/dao/database/DatabaseTest.java +++ b/src/test/java/com/salesforce/dataloader/dao/database/DatabaseTest.java @@ -29,6 +29,7 @@ import com.salesforce.dataloader.controller.Controller; import com.salesforce.dataloader.exception.DataAccessObjectException; import com.salesforce.dataloader.model.Row; +import com.salesforce.dataloader.model.TableRow; import com.salesforce.dataloader.util.AccountRowComparator; import org.apache.logging.log4j.Logger; import com.salesforce.dataloader.util.DLLogManager; @@ -140,7 +141,7 @@ private static void verifyDbInsertOrUpdate(Controller theController, boolean isI reader = new DatabaseReader(theController.getAppConfig(), "queryAccountAll"); reader.open(); int readBatchSize = 1000; - List<Row> readRowList = reader.readRowList(readBatchSize); + List<TableRow> readRowList = reader.readTableRowList(readBatchSize); int rowsProcessed = 0; assertNotNull("Error reading " + readBatchSize + " rows", readRowList); while(readRowList.size() > 0) { @@ -149,18 +150,18 @@ private static void verifyDbInsertOrUpdate(Controller theController, boolean isI Collections.sort(readRowList, new AccountRowComparator(!isInsert)); logger.info("Verifying database success for next " + (rowsProcessed + readRowList.size()) + " rows"); for (int i=0; i < readRowList.size(); i++) { - Row readRow = readRowList.get(i); + TableRow readRow = readRowList.get(i); assertNotNull("Error reading data row #" + i + ": the row shouldn't be null", readRow); - assertTrue("Error reading data row #" + i + ": the row shouldn't be empty", readRow.size() > 0); + assertTrue("Error reading data row #" + i + ": the row shouldn't be empty", readRow.getNonEmptyCellsCount() > 0); Row expectedRow = DatabaseTestUtil.getInsertOrUpdateAccountRow(isInsert, rowsProcessed, DatabaseTestUtil.DateType.VALIDATION); // verify all expected data for (String colName : VALIDATE_COLS) { - verifyCol(colName, readRow, expectedRow); + verifyCol(colName, readRow.convertToRow(), expectedRow); } rowsProcessed++; } - readRowList = reader.readRowList(readBatchSize); + readRowList = reader.readTableRowList(readBatchSize); assertNotNull("Error reading " + readBatchSize + " rows", readRowList); } } finally { diff --git a/src/test/java/com/salesforce/dataloader/process/CsvExtractProcessTest.java b/src/test/java/com/salesforce/dataloader/process/CsvExtractProcessTest.java index 27c70905..33f3fd23 100644 --- a/src/test/java/com/salesforce/dataloader/process/CsvExtractProcessTest.java +++ b/src/test/java/com/salesforce/dataloader/process/CsvExtractProcessTest.java @@ -35,6 +35,7 @@ import com.salesforce.dataloader.exception.DataAccessObjectException; import com.salesforce.dataloader.exception.ProcessInitializationException; import com.salesforce.dataloader.model.Row; +import com.salesforce.dataloader.model.TableRow; import com.sforce.soap.partner.GetUserInfoResult; import com.sforce.soap.partner.SaveResult; import com.sforce.soap.partner.sobject.SObject; @@ -322,7 +323,7 @@ private void testBinaryDataInRTFQueryResult(String inclueRTFBinaryData) throws E final CSVFileReader successRdr = new CSVFileReader(new File(fileName), getController().getAppConfig(), true, false); String idFieldName = this.isBulkV2APIEnabled(insertArgMap)?"sf__Id":"ID"; try { - for (Row row : successRdr.readRowList(Integer.MAX_VALUE)) { + for (TableRow row : successRdr.readTableRowList(Integer.MAX_VALUE)) { final String rowId = (String) row.get(idFieldName); if (rowId != null) { ids.add(rowId); diff --git a/src/test/java/com/salesforce/dataloader/process/CsvHardDeleteTest.java b/src/test/java/com/salesforce/dataloader/process/CsvHardDeleteTest.java index e4ba9896..0ad70a00 100644 --- a/src/test/java/com/salesforce/dataloader/process/CsvHardDeleteTest.java +++ b/src/test/java/com/salesforce/dataloader/process/CsvHardDeleteTest.java @@ -33,6 +33,8 @@ import com.salesforce.dataloader.exception.DataAccessObjectException; import com.salesforce.dataloader.exception.DataAccessObjectInitializationException; import com.salesforce.dataloader.model.Row; +import com.salesforce.dataloader.model.TableRow; + import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; @@ -172,7 +174,7 @@ public InvalidIdTemplateListener(int numValidAccounts) { } @Override - public void updateRow(int idx, Row row) { + public void updateRow(int idx, TableRow row) { if (idx == 0) row.put("ID", "abcde0123456789XYZ"); else @@ -237,7 +239,7 @@ public HeterogeneousIdTemplateListener(int numAccounts, int numContacts) { } @Override - public void updateRow(int idx, Row row) { + public void updateRow(int idx, TableRow row) { if (idx < this.contactIds.length) row.put("ID", this.contactIds[idx]); else diff --git a/src/test/java/com/salesforce/dataloader/process/CsvProcessAttachmentTest.java b/src/test/java/com/salesforce/dataloader/process/CsvProcessAttachmentTest.java index 6ab68d1e..7db773d7 100644 --- a/src/test/java/com/salesforce/dataloader/process/CsvProcessAttachmentTest.java +++ b/src/test/java/com/salesforce/dataloader/process/CsvProcessAttachmentTest.java @@ -36,6 +36,7 @@ import com.salesforce.dataloader.exception.ProcessInitializationException; import com.salesforce.dataloader.exception.UnsupportedOperationException; import com.salesforce.dataloader.model.Row; +import com.salesforce.dataloader.model.TableRow; import com.sforce.soap.partner.QueryResult; import com.sforce.soap.partner.SaveResult; import com.sforce.soap.partner.sobject.SObject; @@ -181,7 +182,7 @@ public AttachmentTemplateListener() { } @Override - public void updateRow(int idx, Row row) { + public void updateRow(int idx, TableRow row) { // set parent account id row.put("ParentId", getAccountIds()[0]); // make body pathname absolute diff --git a/src/test/java/com/salesforce/dataloader/process/CsvProcessTest.java b/src/test/java/com/salesforce/dataloader/process/CsvProcessTest.java index 5073871b..3e14012d 100644 --- a/src/test/java/com/salesforce/dataloader/process/CsvProcessTest.java +++ b/src/test/java/com/salesforce/dataloader/process/CsvProcessTest.java @@ -51,6 +51,7 @@ import com.salesforce.dataloader.exception.DataAccessObjectException; import com.salesforce.dataloader.exception.ProcessInitializationException; import com.salesforce.dataloader.model.Row; +import com.salesforce.dataloader.model.TableRow; import com.salesforce.dataloader.util.AppUtil; import com.sforce.soap.partner.sobject.SObject; @@ -367,7 +368,7 @@ private SObject[] retrieveAccounts(Controller resultController, String... accoun final CSVFileReader successRdr = new CSVFileReader(new File(fileName), getController().getAppConfig(), true, false); try { // TODO: revise the use of Integer.MAX_VALUE - for (Row row : successRdr.readRowList(Integer.MAX_VALUE)) { + for (TableRow row : successRdr.readTableRowList(Integer.MAX_VALUE)) { final String rowId = (String) row.get("ID"); if (rowId != null) { ids.add(rowId); @@ -446,7 +447,7 @@ public AttachmentTemplateListener() { } @Override - public void updateRow(int idx, Row row) { + public void updateRow(int idx, TableRow row) { // set parent account id row.put("ParentId", getAccountIds()[0]); // make body pathname absolute diff --git a/src/test/java/com/salesforce/dataloader/process/DatabaseProcessTest.java b/src/test/java/com/salesforce/dataloader/process/DatabaseProcessTest.java index 66fb927a..3973f57b 100644 --- a/src/test/java/com/salesforce/dataloader/process/DatabaseProcessTest.java +++ b/src/test/java/com/salesforce/dataloader/process/DatabaseProcessTest.java @@ -47,7 +47,7 @@ import com.salesforce.dataloader.dao.database.DatabaseReader; import com.salesforce.dataloader.dao.database.DatabaseTestUtil; import com.salesforce.dataloader.exception.*; -import com.salesforce.dataloader.model.Row; +import com.salesforce.dataloader.model.TableRow; /** * Automated tests for dataloader database batch interface @@ -204,14 +204,14 @@ private void verifyDbSuccess(Controller theController, String dbConfigName, int reader = new DatabaseReader(theController.getAppConfig(), dbConfigName); reader.open(); int readBatchSize = theController.getAppConfig().getInt(AppConfig.PROP_DAO_READ_BATCH_SIZE); - List<Row> successRows = reader.readRowList(readBatchSize); + List<TableRow> successRows = reader.readTableRowList(readBatchSize); int rowsProcessed = 0; assertNotNull("Error reading " + readBatchSize + " rows", successRows); while(successRows.size() > 0) { rowsProcessed += successRows.size(); logger.info("Verifying database success for next " + successRows.size() + " of total " + rowsProcessed + " rows"); assertTrue("No updated rows have been found in the database.", successRows.size() > 0); - successRows = reader.readRowList(readBatchSize); + successRows = reader.readTableRowList(readBatchSize); } assertEquals(expectedSuccesses, rowsProcessed); } catch (DataAccessObjectInitializationException e) { diff --git a/src/test/java/com/salesforce/dataloader/process/ProcessTestBase.java b/src/test/java/com/salesforce/dataloader/process/ProcessTestBase.java index cb6f4db4..8380cc69 100644 --- a/src/test/java/com/salesforce/dataloader/process/ProcessTestBase.java +++ b/src/test/java/com/salesforce/dataloader/process/ProcessTestBase.java @@ -53,6 +53,7 @@ import com.salesforce.dataloader.exception.*; import com.salesforce.dataloader.exception.UnsupportedOperationException; import com.salesforce.dataloader.model.Row; +import com.salesforce.dataloader.model.TableRow; import com.salesforce.dataloader.util.AppUtil; import com.salesforce.dataloader.util.Base64; import com.salesforce.dataloader.action.progress.NihilistProgressAdapter; @@ -88,7 +89,7 @@ protected void verifyErrors(Controller controller, String expectedErrorMessage) final CSVFileReader errReader = new CSVFileReader(new File(fileName), getController().getAppConfig(), true, false); try { errReader.open(); - for (Row errorRow : errReader.readRowList(errReader.getTotalRows())) { + for (TableRow errorRow : errReader.readTableRowList(errReader.getTotalRows())) { String actualMessage = (String) errorRow.get("ERROR"); if (actualMessage == null || !actualMessage.startsWith(expectedErrorMessage)) Assert.fail("Error row does not have the expected error message: " + expectedErrorMessage @@ -109,7 +110,7 @@ protected void verifySuccessIds(Controller ctl, Set<String> ids) throws DataAcce final Set<String> remaining = new HashSet<String>(ids); final Set<String> unexpected = new HashSet<String>(); try { - for (Row row : successRdr.readRowList(Integer.MAX_VALUE)) { + for (TableRow row : successRdr.readTableRowList(Integer.MAX_VALUE)) { final String rowid = (String) row.get("ID"); if (rowid != null && rowid.length() > 0 && !remaining.remove(rowid)) unexpected.add(rowid); } @@ -540,7 +541,7 @@ public String getSOQL(String selectFields) { } protected static interface TemplateListener { - void updateRow(int idx, Row row); + void updateRow(int idx, TableRow row); } /** @@ -555,13 +556,11 @@ public AccountIdTemplateListener(int numAccounts) { } @Override - public void updateRow(int idx, Row row) { + public void updateRow(int idx, TableRow row) { row.put("ID", idx < this.accountIds.length ? this.accountIds[idx] : ""); - if (row.containsValue("standard@org.com")) { - for (String key : row.keySet()) { - if ("standard@org.com".equals(row.get(key))) { - row.put(key, getProperty("test.user.restricted")); - } + for (String key : row.getHeader().getColumns()) { + if ("standard@org.com".equals(row.get(key))) { + row.put(key, getProperty("test.user.restricted")); } } } @@ -591,10 +590,10 @@ protected String convertTemplateToInput(String templateFileName, String inputFil templateReader.open(); int numRows = templateReader.getTotalRows(); - final List<Row> templateRows = templateReader.readRowList(numRows); + final List<TableRow> templateRows = templateReader.readTableRowList(numRows); assertNotNull("CVSReader returned a null list of rows, but expected a list with size " + numRows, templateRows); - final List<Row> inputRows = new ArrayList<Row>(templateRows.size()); + final List<TableRow> inputRows = new ArrayList<TableRow>(templateRows.size()); // verify that the template file is useable assertEquals("Wrong number of rows were read using readRowList while attempting to convert template file: " @@ -603,8 +602,8 @@ protected String convertTemplateToInput(String templateFileName, String inputFil // insert accounts for the whole template or part of it if // maxInserts is smaller then template size int idx = 0; - for (Row templateRow : templateRows) { - final Row row = new Row(templateRow); + for (TableRow templateRow : templateRows) { + final TableRow row = new TableRow(templateRow); if (listeners != null) { for (TemplateListener l : listeners) { l.updateRow(idx, row); @@ -618,7 +617,7 @@ protected String convertTemplateToInput(String templateFileName, String inputFil try { inputWriter.open(); inputWriter.setColumnNames(templateReader.getColumnNames()); - inputWriter.writeRowList(inputRows); + inputWriter.writeTableRowList(inputRows); return inputPath; } finally { inputWriter.close(); diff --git a/src/test/java/com/salesforce/dataloader/util/AccountRowComparator.java b/src/test/java/com/salesforce/dataloader/util/AccountRowComparator.java index b61ad6ad..4998fd1a 100644 --- a/src/test/java/com/salesforce/dataloader/util/AccountRowComparator.java +++ b/src/test/java/com/salesforce/dataloader/util/AccountRowComparator.java @@ -26,7 +26,7 @@ package com.salesforce.dataloader.util; import com.salesforce.dataloader.ConfigTestBase; -import com.salesforce.dataloader.model.Row; +import com.salesforce.dataloader.model.TableRow; import static com.salesforce.dataloader.dao.database.DatabaseTestUtil.NAME_COL; @@ -38,9 +38,9 @@ * @author Alex Warshavsky * @since 8.0 */ -public class AccountRowComparator extends ConfigTestBase implements Comparator<Row> { +public class AccountRowComparator extends ConfigTestBase implements Comparator<TableRow> { - private static String getName(Row o1) { + private static String getName(TableRow o1) { return o1.get(NAME_COL).toString(); } @@ -58,7 +58,7 @@ public AccountRowComparator(boolean isReverse) { } @Override - public int compare(Row o1, Row o2) { + public int compare(TableRow o1, TableRow o2) { final int result = getName(o1).compareTo(getName(o2)); return isReverse ? -result : result; }