Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix duplicate records and inconsistencies in the success log when using 1 batch size #960

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -532,7 +532,7 @@ private void processBatchResults(final BatchInfo batch, final String errorMessag
for (final Row row : rows) {
boolean conversionSuccessOfRow = isRowConversionSuccessful(skippedRowsCount
+ this.firstDAORowForCurrentBatch + dataReaderRowCount++);
if (!conversionSuccessOfRow) {
if (!conversionSuccessOfRow && !controller.getConfig().getBoolean(Config.PROCESS_BULK_CACHE_DATA_FROM_DAO)) {
continue; // this DAO row failed to convert and was not part of the batch sent to the server. Go to the next one
}
final List<String> res = resultRdr.nextRecord();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,11 +124,6 @@ protected boolean isRowConversionSuccessful(int dataSourceRow) {
@Override
public boolean visit(Row row) throws OperationException, DataAccessObjectException,
ConnectionException {
if (controller.getConfig().getBoolean(Config.PROCESS_BULK_CACHE_DATA_FROM_DAO)
|| !controller.getConfig().getBoolean(Config.BULK_API_ENABLED)) {
// either batch mode or cache bulk data uploaded from DAO
this.daoRowList.add(row);
}
// the result are sforce fields mapped to data
Row sforceDataRow = getMapper().mapData(row);
try {
Expand All @@ -142,23 +137,30 @@ public boolean visit(Row row) throws OperationException, DataAccessObjectExcepti
dynaBean.set(fName, value);
}
}
if (controller.getConfig().getBoolean(Config.PROCESS_BULK_CACHE_DATA_FROM_DAO)
|| !controller.getConfig().getBoolean(Config.BULK_API_ENABLED)) {
// either batch mode or cache bulk data uploaded from DAO
this.daoRowList.add(row);
}
dynaArray.add(dynaBean);
this.batchRowToDAORowList.add(this.processedDAORowCounter);
this.processedDAORowCounter++;
} catch (ConversionException | IllegalAccessException conve) {
String errMsg = Messages.getMessage("Visitor", "conversionErrorMsg", conve.getMessage());
getLogger().error(errMsg, conve);

conversionFailed(row, errMsg);
// this row cannot be added since conversion has failed
if (!controller.getConfig().getBoolean(Config.PROCESS_BULK_CACHE_DATA_FROM_DAO)) {
this.processedDAORowCounter++;
}
return false;
} catch (InvocationTargetException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (NoSuchMethodException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
this.processedDAORowCounter++;
}

// load the batch
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,6 @@ private void writeOutputToWriter(Object[] results)
}
for (int i = 0; i < this.daoRowList.size(); i++) {
Row daoRow = this.daoRowList.get(i);
if (!isRowConversionSuccessful(startAtDAORow + i)) {
continue;
}
String statusMsg = null;
if (results instanceof SaveResult[]) {
SaveResult saveRes = (SaveResult)results[batchRowCounter];
Expand Down
Loading