Skip to content

Commit

Permalink
Merge pull request #1393 from ashitsalesforce/master
Browse files Browse the repository at this point in the history 
switch from Row to TableRow
  • Loading branch information
@ashitsalesforce
ashitsalesforce authored Dec 1, 2024
2 parents 78e4538 + d2414e1 commit 32f4672
Show file tree
Hide file tree
Showing 11 changed files with 136 additions and 59 deletions.
2 changes: 0 additions & 2 deletions src/main/java/com/salesforce/dataloader/action/visitor/AbstractQueryVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import com.salesforce.dataloader.action.AbstractExtractAction;
import com.salesforce.dataloader.action.progress.ILoaderProgress;
import com.salesforce.dataloader.client.HttpClientTransport;
import com.salesforce.dataloader.client.HttpTransportInterface;
import com.salesforce.dataloader.config.AppConfig;
import com.salesforce.dataloader.config.Messages;
import com.salesforce.dataloader.controller.Controller;
Expand All @@ -46,7 +45,6 @@
import com.sforce.ws.ConnectionException;

import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
Expand Down
29 changes: 20 additions & 9 deletions src/main/java/com/salesforce/dataloader/config/ConfigPropertyMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@
import com.salesforce.dataloader.dao.csv.CSVFileWriter;
import com.salesforce.dataloader.exception.DataAccessObjectException;
import com.salesforce.dataloader.exception.DataAccessObjectInitializationException;
import com.salesforce.dataloader.model.Row;
import com.salesforce.dataloader.model.TableHeader;
import com.salesforce.dataloader.model.TableRow;
import com.salesforce.dataloader.ui.Labels;
import com.salesforce.dataloader.util.AppUtil;

Expand Down Expand Up @@ -237,11 +238,22 @@ public static void generateCSV(AppConfig appConfig) {
return;
}
try {
ArrayList<String> headerLabelList = new ArrayList<String>();
headerLabelList.add(COL_PROPERTY_NAME);
headerLabelList.add(COL_UI_LABEL);
headerLabelList.add(COL_DESCRIPTION);
headerLabelList.add(COL_DEFAULT_VAL);
headerLabelList.add(COL_IS_READ_ONLY);
headerLabelList.add(COL_IS_COMMAND_LINE_OPTION);
headerLabelList.add(COL_IS_ENCRYPTED);
ArrayList<TableRow> rowList = new ArrayList<TableRow>(propertiesMap.size());
TableHeader header = new TableHeader(headerLabelList);

for (ConfigPropertyMetadata propMD : propertiesMap.values()) {
if (propMD.isInternal()) {
continue;
}
Row row = new Row();
TableRow row = new TableRow(header);
row.put(COL_PROPERTY_NAME, propMD.getName());
row.put(COL_UI_LABEL, propMD.getUiLabelTemplate());
String description = propMD.getDescription();
Expand All @@ -253,13 +265,12 @@ public static void generateCSV(AppConfig appConfig) {
row.put(COL_IS_READ_ONLY, propMD.isReadOnly());
row.put(COL_IS_COMMAND_LINE_OPTION, propMD.isCommandLineOption());
row.put(COL_IS_ENCRYPTED, propMD.isEncrypted());
try {
csvWriter.writeRow(row);
} catch (DataAccessObjectException e) {
logger.warn(Messages.getFormattedString("ConfigPropertyMetadata.errorOutputPropInfo", propMD.getName()));
logger.warn(e.getStackTrace());
continue;
}
rowList.add(row);
}
try {
csvWriter.writeTableRowList(rowList);
} catch (DataAccessObjectException e) {
logger.warn(e.getStackTrace());
}
} finally {
logger.debug(Messages.getFormattedString("ConfigPropertyMetadata.infoGeneratedCSVLocation", getFullPathToPropsFile(appConfig)));
Expand Down
41 changes: 40 additions & 1 deletion src/main/java/com/salesforce/dataloader/dao/csv/CSVFileWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,36 @@ public boolean writeRow(Row row) throws DataAccessObjectException {
}
}

/*
* (non-Javadoc)
* @see com.salesforce.dataloader.dao.csv.Writer#writeRow(java.util.Map)
*/
public boolean writeTableRow(TableRow row) throws DataAccessObjectException {
if (this.columnNames == null || this.columnNames.isEmpty()) {
List<String>colNames = getColumnNamesFromTableRow(row);
this.setColumnNames(colNames);
}
CSVColumnVisitor visitor = new CSVColumnVisitor(fileOut, false, this.columnDelimiter);
try {
visitTableRowColumns(columnNames, row, visitor);
fileOut.newLine();
visitor.newRow();
currentRowNumber++;
return true; // success unless there's an exception
} catch (IOException e) {
logger.error(Messages.getString("CSVWriter.errorWriting"), e); //$NON-NLS-1$
throw new DataAccessObjectException(Messages.getString("CSVWriter.errorWriting"), e); //$NON-NLS-1$
}
}

public List<String> getColumnNamesFromRow(Row row) throws DataAccessObjectInitializationException {
Set<String> fieldNameSet = row.keySet();
return new ArrayList<String>(fieldNameSet);
}

public List<String> getColumnNamesFromTableRow(TableRow row) {
return row.getHeader().getColumns();
}

/*
* (non-Javadoc)
Expand All @@ -222,7 +248,7 @@ public boolean writeTableRowList(List<TableRow> rows) throws DataAccessObjectExc
boolean success = true;
// return the last result, should be same as others
for (TableRow trow : rows) {
success = writeRow(trow.convertToRow());
success = writeTableRow(trow);
}
return success;
}
Expand Down Expand Up @@ -256,6 +282,19 @@ static private void visitColumns(List<String> columnNames, Row row, CSVColumnVis
}
}


static private void visitTableRowColumns(List<String> columnNames, TableRow row, CSVColumnVisitor visitor) throws IOException {
for (String colName : columnNames) {
Object colVal = row.get(colName);
if (colVal == null && colName.contains("(")) {
int lparenIdx = colName.indexOf('(');
int rparenIdx = colName.indexOf(')');
colName = colName.substring(lparenIdx + 1, rparenIdx);
colVal = row.get(colName);
}
visitor.visit(colVal != null ? colVal.toString() : "");
}
}
@Override
public List<String> getColumnNames() {
return columnNames;
Expand Down
7 changes: 7 additions & 0 deletions src/main/java/com/salesforce/dataloader/dao/database/DatabaseWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,13 @@ public boolean writeRow(Row inputRow) throws DataAccessObjectException {
return writeRowList(inputRowList);
}

public boolean writeTableRow(TableRow inputRow) throws DataAccessObjectException {
// FIXME: Think about refactoring this for the caller to writeRow() and here take care of batching internally
List<TableRow> inputRowList = new ArrayList<TableRow>();
inputRowList.add(inputRow);
return writeTableRowList(inputRowList);
}

/**
* @param sqe
*/
Expand Down
14 changes: 2 additions & 12 deletions src/main/java/com/salesforce/dataloader/model/Row.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,31 +42,21 @@
* class and not be spread in multiple class.
*/
public class Row implements Map<String, Object> {

private static final int DEFAULT_COLUMN_COUNT = 16; // same as HashMap
private final Map<String, Object> internalMap;
private final Map<String, String> keyMap = new HashMap<String, String>();

public Row() {
this(DEFAULT_COLUMN_COUNT);
}

public Row(int columnCount) {
internalMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
this.internalMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
}

public Row(Map<String, Object> internalMap) {
this(internalMap.size());
this.internalMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
this.internalMap.putAll(internalMap);
for (String key : internalMap.keySet()) {
this.keyMap.put(key.toLowerCase(), key);
}
}

public static Row emptyRow() {
return new Row(Collections.<String, Object>emptyMap());
}

public static Row singleEntryImmutableRow(String key, Object value) {
return new Row(Collections.singletonMap(key, value));
}
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/com/salesforce/dataloader/model/TableHeader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
*/
package com.salesforce.dataloader.model;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -34,7 +35,7 @@ public class TableHeader {
private int lastColPosition = 0;
private List<String> columns;
public TableHeader(List<String> cols) {
this.columns = cols;
this.columns = new ArrayList<String>(cols);
for (String colName : cols) {
if (colName == null) {
continue;
Expand All @@ -49,6 +50,6 @@ public Integer getColumnPosition(String columnName) {
}

public List<String> getColumns() {
return this.columns;
return new ArrayList<String>(columns);
}
}
19 changes: 12 additions & 7 deletions src/test/java/com/salesforce/dataloader/dao/CsvTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import com.salesforce.dataloader.config.AppConfig;
import com.salesforce.dataloader.dao.csv.CSVFileReader;
import com.salesforce.dataloader.dao.csv.CSVFileWriter;
import com.salesforce.dataloader.model.Row;
import com.salesforce.dataloader.model.TableHeader;
import com.salesforce.dataloader.model.TableRow;
import com.salesforce.dataloader.util.AppUtil;

Expand All @@ -49,8 +49,8 @@ public class CsvTest extends ConfigTestBase {
private static final String COLUMN_2_NAME = "column2";
private static final String COLUMN_3_NAME = "column3";
private List<String> writeHeader;
private Row row1;
private Row row2;
private TableRow row1;
private TableRow row2;

@Before
public void createTestData() {
Expand All @@ -59,12 +59,17 @@ public void createTestData() {
writeHeader.add("COL2");
writeHeader.add("COL3");

row1 = new Row();
ArrayList<String> headerLabelList = new ArrayList<String>();
headerLabelList.add("COL1");
headerLabelList.add("COL2");
headerLabelList.add("COL3");
TableHeader header = new TableHeader(headerLabelList);
row1 = new TableRow(header);
row1.put("COL1", "row1col1");
row1.put("COL2", "row1col2");
row1.put("COL3", "row1col3");

row2 = new Row();
row2 = new TableRow(header);
row2.put("COL1", "row2col1");
row2.put("COL2", "row2col2");
row2.put("COL3", "row2col3");
Expand Down Expand Up @@ -143,15 +148,15 @@ private void doTestCSVWriteBasic(String delimiter) throws Exception {
File f = new File(getTestDataDir(), "csvtestTemp.csv");
String path = f.getAbsolutePath();
CSVFileWriter writer = new CSVFileWriter(path, getController().getAppConfig(), delimiter);
List<Row> rowList = new ArrayList<Row>();
List<TableRow> rowList = new ArrayList<TableRow>();

rowList.add(row1);
rowList.add(row2);

writer.open();
writer.setColumnNames(writeHeader);

writer.writeRowList(rowList);
writer.writeTableRowList(rowList);
writer.close();

compareWriterFile(path, delimiter, false, false); // 3rd param false and 4th param false => CSV for a upload
Expand Down
8 changes: 4 additions & 4 deletions src/test/java/com/salesforce/dataloader/dao/database/DatabaseTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,14 @@
import com.salesforce.dataloader.ConfigTestBase;
import com.salesforce.dataloader.controller.Controller;
import com.salesforce.dataloader.exception.DataAccessObjectException;
import com.salesforce.dataloader.model.Row;
import com.salesforce.dataloader.model.TableRow;
import com.salesforce.dataloader.util.AccountRowComparator;
import org.apache.logging.log4j.Logger;
import com.salesforce.dataloader.util.DLLogManager;

import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;

import java.sql.Timestamp;
Expand Down Expand Up @@ -153,10 +153,10 @@ private static void verifyDbInsertOrUpdate(Controller theController, boolean isI
TableRow readRow = readRowList.get(i);
assertNotNull("Error reading data row #" + i + ": the row shouldn't be null", readRow);
assertTrue("Error reading data row #" + i + ": the row shouldn't be empty", readRow.getNonEmptyCellsCount() > 0);
Row expectedRow = DatabaseTestUtil.getInsertOrUpdateAccountRow(isInsert, rowsProcessed, DatabaseTestUtil.DateType.VALIDATION);
TableRow expectedRow = DatabaseTestUtil.getInsertOrUpdateAccountRow(isInsert, rowsProcessed, DatabaseTestUtil.DateType.VALIDATION);
// verify all expected data
for (String colName : VALIDATE_COLS) {
verifyCol(colName, readRow.convertToRow(), expectedRow);
verifyCol(colName, readRow, expectedRow);
}

rowsProcessed++;
Expand All @@ -169,7 +169,7 @@ private static void verifyDbInsertOrUpdate(Controller theController, boolean isI
}
}

private static void verifyCol(String colName, Row row, Row expectedRow) {
private static void verifyCol(String colName, TableRow row, TableRow expectedRow) {
Object actualValue = row.get(colName);
Object expectedValue = expectedRow.get(colName);
assertNotNull("actual value is null", actualValue);
Expand Down
28 changes: 20 additions & 8 deletions src/test/java/com/salesforce/dataloader/dao/database/DatabaseTestUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@
import com.salesforce.dataloader.controller.Controller;
import com.salesforce.dataloader.exception.DataAccessObjectException;
import com.salesforce.dataloader.exception.DataAccessObjectInitializationException;
import com.salesforce.dataloader.model.Row;
import com.salesforce.dataloader.model.TableHeader;
import com.salesforce.dataloader.model.TableRow;

import junit.framework.TestCase;
import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -99,16 +101,16 @@ public static void insertOrUpdateAccountsDb(Controller theController, boolean is
sqlConfig.getSqlParams().put(LAST_UPDATED_COL, dateClass.getName());
writer = new DatabaseWriter(theController.getAppConfig(), dbConfigName, dataSource, sqlConfig);
writer.open();
List<Row> accountRowList = new ArrayList<Row>();
List<TableRow> accountRowList = new ArrayList<TableRow>();
int rowsProcessed = 0;
for(int i=0; i < numAccounts; i++) {
Row accountRow = getInsertOrUpdateAccountRow(isInsert, i, dateType, insertNulls);
TableRow accountRow = getInsertOrUpdateAccountRow(isInsert, i, dateType, insertNulls);
accountRowList.add(accountRow);
if(accountRowList.size() >= 1000 || i == (numAccounts-1)) {
rowsProcessed += accountRowList.size();
writer.writeRowList(accountRowList);
writer.writeTableRowList(accountRowList);
logger.info("Written " + rowsProcessed + " of " + numAccounts + " total accounts using database config: " + dbConfigName);
accountRowList = new ArrayList<Row>();
accountRowList = new ArrayList<TableRow>();
}
}
} catch (DataAccessObjectInitializationException e) {
Expand All @@ -121,7 +123,7 @@ public static void insertOrUpdateAccountsDb(Controller theController, boolean is
}
}

public static Row getInsertOrUpdateAccountRow(boolean isInsert, int seqNum, DateType dateType) {
public static TableRow getInsertOrUpdateAccountRow(boolean isInsert, int seqNum, DateType dateType) {
return getInsertOrUpdateAccountRow(isInsert, seqNum, dateType, false);
}

Expand All @@ -142,8 +144,18 @@ public static DatabaseConfig getDatabaseConfig(Controller controller, String dbC
* @param dateType Type for the date field values
* @return Row containing account data based on seqNum
*/
public static Row getInsertOrUpdateAccountRow(boolean isInsert, int seqNum, DateType dateType, boolean insertNulls) {
Row row = new Row();
public static TableRow getInsertOrUpdateAccountRow(boolean isInsert, int seqNum, DateType dateType, boolean insertNulls) {
ArrayList<String> headerLabelList = new ArrayList<String>();
headerLabelList.add(EXT_ID_COL);
headerLabelList.add(NAME_COL);
headerLabelList.add(SFDC_ID_COL);
headerLabelList.add(ACCOUNT_NUMBER_COL);
headerLabelList.add(PHONE_COL);
headerLabelList.add(REVENUE_COL);
headerLabelList.add(LAST_UPDATED_COL);

TableHeader header = new TableHeader(headerLabelList);
TableRow row = new TableRow(header);
String operation;
int seqInt;
// external id is the key, use normal sequencing for update so the same set of records gets updated as inserted
Expand Down
Loading

0 comments on commit 32f4672

Please sign in to comment.