Skip to content

Commit

Permalink
increase TableRow direct use - replace readRow() with readTableRow()
Browse files Browse the repository at this point in the history 
increase TableRow direct use - replace readRow() with readTableRow() in DataReader interface and its implementations - CsvFileReader and DatabaeReader.
  • Loading branch information
@ashitsalesforce
ashitsalesforce committed Nov 30, 2024
1 parent 6a4b8d4 commit 159870c
Show file tree
Hide file tree
Showing 12 changed files with 64 additions and 106 deletions.
9 changes: 0 additions & 9 deletions src/main/java/com/salesforce/dataloader/dao/DataReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,6 @@
* @since 8.0
*/
public interface DataReader extends DataAccessObject {

/**
* Get a row of data from a data source
*
* @return a {@link Row} containing all the keys and values of a row
* @throws DataAccessObjectException
*/
Row readRow() throws DataAccessObjectException;

/**
* Get a row of data from a data source
*
Expand Down
10 changes: 0 additions & 10 deletions src/main/java/com/salesforce/dataloader/dao/csv/CSVFileReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
import com.salesforce.dataloader.exception.DataAccessObjectException;
import com.salesforce.dataloader.exception.DataAccessObjectInitializationException;
import com.salesforce.dataloader.exception.DataAccessRowException;
import com.salesforce.dataloader.model.Row;
import com.salesforce.dataloader.model.TableHeader;
import com.salesforce.dataloader.model.TableRow;
import com.salesforce.dataloader.util.AppUtil;
Expand Down Expand Up @@ -205,15 +204,6 @@ record = csvReader.nextRecord();
}
return trow;
}

@Override
public Row readRow() throws DataAccessObjectException {
TableRow tableRow = readTableRow();
if (tableRow == null) {
return null;
}
return tableRow.convertToRow();
}

/**
* @return Names of output columns being read during each readRow call
Expand Down
10 changes: 0 additions & 10 deletions src/main/java/com/salesforce/dataloader/dao/database/DatabaseReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import java.sql.*;
import java.util.*;

import com.salesforce.dataloader.model.Row;
import com.salesforce.dataloader.model.TableHeader;
import com.salesforce.dataloader.model.TableRow;

Expand Down Expand Up @@ -217,15 +216,6 @@ protected TableRow readTableRowFromDAO() throws DataAccessObjectException {
throw new DataAccessObjectException(errMsg, e);
}
}

@Override
public Row readRow() throws DataAccessObjectException {
TableRow tableRow = readTableRow();
if (tableRow == null) {
return null;
}
return tableRow.convertToRow();
}

@Override
public List<String> getColumnNames() {
Expand Down
17 changes: 8 additions & 9 deletions src/main/java/com/salesforce/dataloader/ui/CSVViewerDialog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.ArrayList;
import java.util.List;

import org.apache.logging.log4j.Logger;
Expand All @@ -57,7 +56,7 @@
import com.salesforce.dataloader.dao.csv.CSVFileReader;
import com.salesforce.dataloader.exception.DataAccessObjectException;
import com.salesforce.dataloader.exception.DataAccessObjectInitializationException;
import com.salesforce.dataloader.model.Row;
import com.salesforce.dataloader.model.TableRow;
import com.salesforce.dataloader.ui.csvviewer.CSVContentProvider;
import com.salesforce.dataloader.ui.csvviewer.CSVLabelProvider;
import com.salesforce.dataloader.util.AppUtil;
Expand Down Expand Up @@ -292,23 +291,23 @@ private void updateCSVTable(CSVFileReader csvReader) {

List<List<Object>> rowList = new ArrayList<List<Object>>();
for (int i = 0; i < numberOfRows; i++) {
Row rowMap;
TableRow row;
try {
rowMap = csvReader.readRow();
row = csvReader.readTableRow();
} catch (DataAccessObjectException e) {
break;
}
if (!DAORowUtil.isValidRow(rowMap)) {
if (!DAORowUtil.isValidTableRow(row)) {
break;
}

List<String> columns = csvReader.getColumnNames();
List<Object> row = new ArrayList<Object>();
row.add(0, String.valueOf(i + 1));
List<Object> listOfRowCells = new ArrayList<Object>();
listOfRowCells.add(0, String.valueOf(i + 1));
for(String column : columns) {
row.add(rowMap.get(column));
listOfRowCells.add(row.get(column));
}
rowList.add(row);
rowList.add(listOfRowCells);
}
csvReader.close();
csvTblViewer.setInput(rowList);
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/salesforce/dataloader/util/DAORowUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ private void rowToStart(AppConfig cfg, DataReader daoReader) throws DataAccessOb
// keep skipping over rows until we run into an invalid row or we have gotten
// to the starting row
while (daoReader.getCurrentRowNumber() < rowToStart) {
if (!DAORowUtil.isValidRow(daoReader.readRow())) break;
if (!DAORowUtil.isValidTableRow(daoReader.readTableRow())) break;
}
}
}
Expand Down
29 changes: 15 additions & 14 deletions src/test/java/com/salesforce/dataloader/dao/CsvTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.salesforce.dataloader.dao.csv.CSVFileReader;
import com.salesforce.dataloader.dao.csv.CSVFileWriter;
import com.salesforce.dataloader.model.Row;
import com.salesforce.dataloader.model.TableRow;
import com.salesforce.dataloader.util.AppUtil;

import static org.junit.Assert.assertEquals;
Expand Down Expand Up @@ -105,12 +106,12 @@ private void testCSVReadBasic(String csvFile) throws Exception {
assertEquals(COLUMN_2_NAME, headerRow.get(1));
assertEquals(COLUMN_3_NAME, headerRow.get(2));

Row firstRow = csv.readRow();
TableRow firstRow = csv.readTableRow();
assertEquals("row1-1", firstRow.get(COLUMN_1_NAME));
assertEquals("row1-2", firstRow.get(COLUMN_2_NAME));
assertEquals("row1-3", firstRow.get(COLUMN_3_NAME));

Row secondRow = csv.readRow();
TableRow secondRow = csv.readTableRow();
assertEquals("row2-1", secondRow.get(COLUMN_1_NAME));
assertEquals("row2-2", secondRow.get(COLUMN_2_NAME));
assertEquals("row2-3", secondRow.get(COLUMN_3_NAME));
Expand Down Expand Up @@ -170,19 +171,19 @@ public void testReadingSeparatedValues () throws Exception {

CSVFileReader csv = new CSVFileReader(f, getController().getAppConfig(), false, false);
csv.open();
Row firstRow = csv.readRow();
TableRow firstRow = csv.readTableRow();
assertEquals("somev1", firstRow.get("some"));
assertEquals(4, firstRow.size());
Row secondRow = csv.readRow();
assertEquals(4, firstRow.getNonEmptyCellsCount());
TableRow secondRow = csv.readTableRow();
assertEquals("somev2", secondRow.get("some"));
csv.close();

getController().getAppConfig().setValue("loader.csvOther", false);
csv = new CSVFileReader(f, getController().getAppConfig(), false, false);
csv.open();
firstRow = csv.readRow();
firstRow = csv.readTableRow();
assertEquals("col12!somev1", firstRow.get("column2!some"));
assertEquals(3, firstRow.size());
assertEquals(3, firstRow.getNonEmptyCellsCount());
csv.close();
}

Expand All @@ -195,10 +196,10 @@ public void testReadingEscapedValues() throws Exception {
CSVFileReader csv = new CSVFileReader(f, getController().getAppConfig(), false, false);
csv.open();

Row firstRow = csv.readRow();
TableRow firstRow = csv.readTableRow();
assertEquals("\"The Best\" Account", firstRow.get(COLUMN_1_NAME));

Row secondRow = csv.readRow();
TableRow secondRow = csv.readTableRow();
assertEquals("The \"Best\" Account", secondRow.get(COLUMN_1_NAME));

csv.close();
Expand All @@ -210,7 +211,7 @@ public void testCsvWithManyRowsCanBeParsed() throws Exception {
csvFileReader.open();
assertEquals(20000, csvFileReader.getTotalRows());
int count = 0;
for(Row row = csvFileReader.readRow(); row != null; row = csvFileReader.readRow(), count++);
for(TableRow row = csvFileReader.readTableRow(); row != null; row = csvFileReader.readTableRow(), count++);
assertEquals(20000, count);
}

Expand Down Expand Up @@ -262,15 +263,15 @@ private void compareWriterFile(String filePath, String delimiterStr, boolean ign
}

//check that row 1 is valid
Row firstRow = csv.readRow();
TableRow firstRow = csv.readTableRow();
for (String headerColumn : writeHeader) {
assertEquals(row1.get(headerColumn), firstRow.get(headerColumn));
assertEquals(row1.get(headerColumn), firstRow.get(headerColumn));
}

//check that row 2 is valid
Row secondRow = csv.readRow();
TableRow secondRow = csv.readTableRow();
for (String headerColumn : writeHeader) {
assertEquals(row2.get(headerColumn), secondRow.get(headerColumn));
assertEquals(row2.get(headerColumn), secondRow.get(headerColumn));
}
csv.close();
if (isQueryResultsCSV) {
Expand Down
10 changes: 5 additions & 5 deletions src/test/java/com/salesforce/dataloader/process/CsvExtractProcessTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,17 +181,17 @@ public void testPolymorphicRelationshipExtract() throws Exception {
// open the results of the extraction
final CSVFileReader rdr = new CSVFileReader(new File(argmap.get(AppConfig.PROP_DAO_NAME)), getController().getAppConfig(), true, false);
rdr.open();
Row row = rdr.readRow();
TableRow row = rdr.readTableRow();
assertNotNull(row);
assertEquals(5,row.size());
assertEquals(5,row.getNonEmptyCellsCount());
// validate the extract results are correct.
assertEquals(leadidArr[0], row.get("LID"));
assertTrue(userInfo.getUserFullName().contains(row.get("LNAME").toString()));
assertEquals(userInfo.getUserFullName(), row.get("NAME__RESULT"));
assertEquals(uid, row.get("OID"));
assertEquals(uid,row.get("OWNID"));
// validate that we have read the only result. there should be only one.
assertNull(rdr.readRow());
assertNull(rdr.readTableRow());
} finally {
// cleanup here since the parent doesn't clean up leads
getBinding().delete(leadidArr);
Expand Down Expand Up @@ -342,13 +342,13 @@ private void testBinaryDataInRTFQueryResult(String inclueRTFBinaryData) throws E
controller = runProcess(queryArgMap, 1);
CSVFileReader queryResultsReader = new CSVFileReader(new File(queryArgMap.get(AppConfig.PROP_DAO_NAME)), getController().getAppConfig(), true, false);
queryResultsReader.open();
Row queryResultsRow = queryResultsReader.readRow();
TableRow queryResultsRow = queryResultsReader.readTableRow();
String queryResultsRTVal = (String)queryResultsRow.get("RICHTEXT__c");

if ("true".equalsIgnoreCase(inclueRTFBinaryData)) {
CSVFileReader uploadedCSVReader = new CSVFileReader(new File(getTestDataDir() + "/acctsWithBinaryDataInRTF.csv"), getController().getAppConfig(), true, false);
uploadedCSVReader.open();
Row uploadedRow = uploadedCSVReader.readRow();
TableRow uploadedRow = uploadedCSVReader.readTableRow();
String uploadedRTVal = (String)queryResultsRow.get("RICHTEXT__c");
assertEquals("Binary data in query result file does not match uploaded data: "
+ queryArgMap.get(AppConfig.PROP_DAO_NAME), queryResultsRTVal, uploadedRTVal);
Expand Down
4 changes: 2 additions & 2 deletions src/test/java/com/salesforce/dataloader/process/CsvProcessTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ public void testInsertAccountWithMultipleBatchesCSV() throws ProcessInitializati
String successFileName = controller.getAppConfig().getString(AppConfig.PROP_OUTPUT_SUCCESS);
File successFile = new File(successFileName);
CSVFileReader csvReader = new CSVFileReader(successFile, controller.getAppConfig(), false, false);
Row row1 = csvReader.readRow();
Row row2 = csvReader.readRow();
TableRow row1 = csvReader.readTableRow();
TableRow row2 = csvReader.readTableRow();
String oracleIdRow1 = (String)row1.get("oracle_id");
String oracleIdRow2 = (String)row2.get("oracle_id");
if (oracleIdRow1 == null) {
Expand Down
56 changes: 22 additions & 34 deletions src/test/java/com/salesforce/dataloader/process/CsvProcessWithOffsetTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import com.salesforce.dataloader.exception.DataAccessObjectException;
import com.salesforce.dataloader.exception.DataAccessObjectInitializationException;
import com.salesforce.dataloader.model.Row;
import com.salesforce.dataloader.model.TableRow;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
Expand Down Expand Up @@ -173,30 +174,30 @@ private void verifyOffsetFromInputAndOutputFiles(int numberOfOffsetRows, AppConf
}

// Initializations of row results
Row firstInputOffsetAdjustedRow = new Row();
Row lastInputRow = new Row();
Row firstSuccessRow = new Row();
Row lastSuccessRow = new Row();
Row firstErrorRow = new Row();
Row lastErrorRow = new Row();
TableRow firstInputOffsetAdjustedRow = null;
TableRow lastInputRow = null;
TableRow firstSuccessRow = null;
TableRow lastSuccessRow = null;
TableRow firstErrorRow = null;
TableRow lastErrorRow = null;

// The next few if statements deal with the edge statements on file size...(i.e. suppose that there are no
// errors)
if (numberOfSuccessRows > 0) {
getFirstRow(firstSuccessRow, successFileReader, true, 0);
getLastRow(lastSuccessRow, successFileReader, true);
firstSuccessRow = getFirstRow(successFileReader, true, 0);
lastSuccessRow = getLastRow(successFileReader, true);
}

if (numberOfErrorRows > 0) {
getFirstRow(firstErrorRow, errorFileReader, false, 0);
getLastRow(lastErrorRow, errorFileReader, false);
firstErrorRow = getFirstRow(errorFileReader, false, 0);
lastErrorRow = getLastRow(errorFileReader, false);
}

if (numberOfInputRows > 0) {
final CSVFileReader inputFileReader = openConfiguredPath(cfg, AppConfig.PROP_DAO_NAME);

getFirstRow(firstInputOffsetAdjustedRow, inputFileReader, false, numberOfOffsetRows);
getLastRow(lastInputRow, inputFileReader, false);
firstInputOffsetAdjustedRow = getFirstRow(inputFileReader, false, numberOfOffsetRows);
lastInputRow = getLastRow(inputFileReader, false);
}

// Requirement I: First offset-adjusted row of input matches to either the error or success file's first row
Expand Down Expand Up @@ -226,40 +227,27 @@ private CSVFileReader openConfiguredPath(AppConfig cfg, String configSetting)
return rdr;
}

private void getFirstRow(Row rowResult, CSVFileReader reader, boolean isSuccessFile, int rowOffset)
private TableRow getFirstRow(CSVFileReader reader, boolean isSuccessFile, int rowOffset)
throws Exception {
Row firstRow = reader.readRow();
TableRow firstRow = reader.readTableRow();

for (int i = 0; i < rowOffset; i++) {
firstRow = reader.readRow(); // then, for each, move down one row
}

if (isSuccessFile) {
// Also ask for ID
rowResult.put("ID", firstRow.get("ID"));
}
if (firstRow != null && firstRow.get("NAME") != null) {
rowResult.put("NAME", firstRow.get("NAME"));
firstRow = reader.readTableRow(); // then, for each, move down one row
}
return firstRow;
}

private void getLastRow(Row rowResult, CSVFileReader reader, boolean isSuccessFile)
private TableRow getLastRow(CSVFileReader reader, boolean isSuccessFile)
throws Exception {

Row tempRow = new Row();
Row lastRow = new Row();
TableRow tempRow = null;
TableRow lastRow = null;

// get to the last row:
while ((tempRow = reader.readRow()) != null) {
while ((tempRow = reader.readTableRow()) != null) {
lastRow = tempRow;
}

if (isSuccessFile) {
// Also ask for ID
rowResult.put("ID", lastRow.get("ID"));
}

rowResult.put("NAME", lastRow.get("NAME"));
return lastRow;
}

private Map<String, String> getRowOffsetTestConfig(Object offset, int numInserts) throws DataAccessObjectException {
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/com/salesforce/dataloader/process/NAProcessTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ private String getCsvFieldValue(String csvFile, String fieldName) throws Excepti
CSVFileReader reader = new CSVFileReader(new File(csvFile), getController().getAppConfig(), true, false);
reader.open();
assertEquals(1, reader.getTotalRows());
String fieldValue = (String)reader.readRow().get(fieldName);
String fieldValue = (String)reader.readTableRow().get(fieldName);
reader.close();
return fieldValue;
}
Expand Down
Loading

0 comments on commit 159870c

Please sign in to comment.