Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

configurable caching of global sobject describe and sobject fields #948

Merged
merged 1 commit into from
Jan 15, 2024
Merged

configurable caching of global sobject describe and sobject fields #948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 35 additions & 46 deletions src/main/java/com/salesforce/dataloader/client/PartnerClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,10 +223,10 @@ public DescribeSObjectResult run(String entity) throws ConnectionException {
}
};

private DescribeGlobalResult entityTypes;
private final Map<String, DescribeRefObject> referenceDescribes = new HashMap<String, DescribeRefObject>();
private final Map<String, DescribeGlobalSObjectResult> describeGlobalResults = new HashMap<String, DescribeGlobalSObjectResult>();
private final Map<String, DescribeSObjectResult> entityDescribes = new HashMap<String, DescribeSObjectResult>();
private DescribeGlobalResult describeGlobalResults;
private final Map<String, DescribeRefObject> referenceEntitiesDescribesMap = new HashMap<String, DescribeRefObject>();
private final Map<String, DescribeGlobalSObjectResult> describeGlobalResultsMap = new HashMap<String, DescribeGlobalSObjectResult>();
private final Map<String, DescribeSObjectResult> entityFieldDescribesMap = new HashMap<String, DescribeSObjectResult>();

private final boolean enableRetries;
private final int maxRetries;
Expand Down Expand Up @@ -526,15 +526,31 @@ public PartnerConnection getClient() {
}

public Map<String, DescribeGlobalSObjectResult> getDescribeGlobalResults() {
return describeGlobalResults;
}

Map<String, DescribeSObjectResult> getEntityDescribeMap() {
return this.entityDescribes;
if (this.describeGlobalResults == null || !config.getBoolean(Config.CACHE_DESCRIBE_GLOBAL_RESULTS)) {
this.describeGlobalResultsMap.clear();
try {
this.describeGlobalResults = runOperation(DESCRIBE_GLOBAL_OPERATION, null);
} catch (ConnectionException e) {
logger.error("Failed to get description of sobjects", e.getMessage());
return null;
}
}

if (this.describeGlobalResultsMap.isEmpty()) {
for (DescribeGlobalSObjectResult res : describeGlobalResults.getSobjects()) {
if (res != null) {
if (res.getLabel().startsWith("__MISSING LABEL__")) {
res.setLabel(res.getName());
}
this.describeGlobalResultsMap.put(res.getName(), res);
}
}
}
return describeGlobalResultsMap;
}

DescribeGlobalResult getEntityTypes() {
return entityTypes;
private Map<String, DescribeSObjectResult> getCachedEntityDescribeMap() {
return this.entityFieldDescribesMap;
}

public DescribeSObjectResult getFieldTypes() {
Expand All @@ -547,7 +563,7 @@ public DescribeSObjectResult getFieldTypes() {
}

public Map<String, DescribeRefObject> getReferenceDescribes() {
return referenceDescribes;
return referenceEntitiesDescribesMap;
}

public LimitInfo getAPILimitInfo() {
Expand Down Expand Up @@ -749,43 +765,13 @@ private void retrySleep(String operationName, int retryNum) {
}
}

/**
* Gets the sObject describes for all entities
*/
public boolean setEntityDescribes() throws ConnectionException {
setEntityTypes();
if (this.describeGlobalResults.isEmpty()) {
for (DescribeGlobalSObjectResult res : entityTypes.getSobjects()) {
if (res != null) {
if (res.getLabel().startsWith("__MISSING LABEL__")) {
res.setLabel(res.getName());
}
this.describeGlobalResults.put(res.getName(), res);
}
}
}

return true;
}

/**
* Gets the available objects from the global describe
*/
private void setEntityTypes() throws ConnectionException {
if (this.entityTypes == null)
this.entityTypes = runOperation(DESCRIBE_GLOBAL_OPERATION, null);
}

/**
* Set the map of references to object external id info for current entity
*
* @throws ConnectionException
*/
public void setFieldReferenceDescribes() throws ConnectionException {
referenceDescribes.clear();
if (getDescribeGlobalResults().isEmpty()) {
setEntityDescribes();
}
referenceEntitiesDescribesMap.clear();
if (getFieldTypes() == null) {
setFieldTypes();
}
Expand Down Expand Up @@ -832,7 +818,7 @@ private void processParentObjectForLookupReferences(String parentObjectName, Fie
}
if (!parentIdLookupFieldMap.isEmpty()) {
DescribeRefObject describeRelationship = new DescribeRefObject(parentObjectName, childObjectField, parentIdLookupFieldMap);
referenceDescribes.put(childObjectField.getRelationshipName(), describeRelationship);
referenceEntitiesDescribesMap.put(childObjectField.getRelationshipName(), describeRelationship);
}
}

Expand Down Expand Up @@ -918,11 +904,14 @@ private String getDefaultServer() {
*/

public DescribeSObjectResult describeSObject(String entity) throws ConnectionException {
DescribeSObjectResult result = getEntityDescribeMap().get(entity);
DescribeSObjectResult result = null;
if (config.getBoolean(Config.CACHE_DESCRIBE_GLOBAL_RESULTS)) {
result = getCachedEntityDescribeMap().get(entity);
}
if (result == null) {
result = runOperation(DESCRIBE_SOBJECT_OPERATION, entity);
if (result != null) {
getEntityDescribeMap().put(result.getName(), result);
getCachedEntityDescribeMap().put(result.getName(), result);
}
}
return result;
Expand Down
4 changes: 3 additions & 1 deletion src/main/java/com/salesforce/dataloader/config/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,8 @@ public class Config {
public static final String CSV_DELIMITER_OTHER_VALUE = "loader.csvOtherValue";
public static final String CSV_DELIMITER_FOR_QUERY_RESULTS = "loader.query.delimiter";
public static final String BUFFER_UNPROCESSED_BULK_QUERY_RESULTS = "loader.bufferUnprocessedBulkQueryResults";

public static final String CACHE_DESCRIBE_GLOBAL_RESULTS = "loader.cacheSObjectNamesAndFields";

//Special Internal Configs
public static final String SFDC_INTERNAL = "sfdcInternal"; //$NON-NLS-1$
public static final String SFDC_INTERNAL_IS_SESSION_ID_LOGIN = "sfdcInternal.isSessionIdLogin"; //$NON-NLS-1$
Expand Down Expand Up @@ -606,6 +607,7 @@ private void setDefaults() {
setDefaultValue(DAO_WRITE_POSTPROCESSOR_SCRIPT, "");
setDefaultValue(LIMIT_OUTPUT_TO_QUERY_FIELDS, true);
setDefaultValue(WIZARD_CLOSE_ON_FINISH, true);
setDefaultValue(CACHE_DESCRIBE_GLOBAL_RESULTS, true);
}

/**
Expand Down
5 changes: 0 additions & 5 deletions src/main/java/com/salesforce/dataloader/controller/Controller.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,11 +186,6 @@ private boolean loginIfSessionExists(ClientBase<?> clientToLogin) {
public boolean loginIfSessionExists() {
return loginIfSessionExists(getClient());
}

public boolean setEntityDescribes() throws ConnectionException {
validateSession();
return getPartnerClient().setEntityDescribes();
}

public static void setAPIVersion(String apiVersionStr) {
API_VERSION = apiVersionStr;
Expand Down
13 changes: 12 additions & 1 deletion src/main/java/com/salesforce/dataloader/ui/AdvancedSettingsDialog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ public class AdvancedSettingsDialog extends BaseDialog {
private Button buttonTruncateFields;
private Button buttonFormatPhoneFields;
private Button buttonKeepAccountTeam;
private Button buttonCacheDescribeGlobalResults;
private Button buttonUseBulkApi;
private Button buttonUseBulkV2Api;
private Button buttonBulkApiSerialMode;
Expand Down Expand Up @@ -500,6 +501,15 @@ public void verifyText(VerifyEvent event) {
data.widthHint = 5 * textSize.x;
textQueryResultsDelimiterValue.setLayoutData(data);

Label labelCacheDescribeGlobalResults = new Label(restComp, SWT.RIGHT | SWT.WRAP);
labelCacheDescribeGlobalResults.setText(Labels.getString("AdvancedSettingsDialog.cacheDescribeGlobalResults"));
data = new GridData(GridData.HORIZONTAL_ALIGN_END);
labelCacheDescribeGlobalResults.setLayoutData(data);

boolean cacheDescribeGlobalResults = config.getBoolean(Config.CACHE_DESCRIBE_GLOBAL_RESULTS);
buttonCacheDescribeGlobalResults = new Button(restComp, SWT.CHECK);
buttonCacheDescribeGlobalResults.setSelection(cacheDescribeGlobalResults);

// Keep Account team setting
Label labelKeepAccountTeam = new Label(restComp, SWT.RIGHT | SWT.WRAP);
labelKeepAccountTeam.setText(Labels.getString("AdvancedSettingsDialog.keepAccountTeam"));
Expand All @@ -520,7 +530,7 @@ public void widgetSelected(SelectionEvent e) {
});
buttonKeepAccountTeam.setToolTipText(Labels.getString("AdvancedSettingsDialog.keepAccountTeamHelp"));
labelKeepAccountTeam.setToolTipText(Labels.getString("AdvancedSettingsDialog.keepAccountTeamHelp"));

// Enable Bulk API Setting
Label labelUseBulkApi = new Label(restComp, SWT.RIGHT | SWT.WRAP);
labelUseBulkApi.setText(Labels.getString("AdvancedSettingsDialog.useBulkApi")); //$NON-NLS-1$
Expand Down Expand Up @@ -886,6 +896,7 @@ public void widgetSelected(SelectionEvent event) {
config.setValue(Config.PROXY_USERNAME, textProxyUsername.getText());
config.setValue(Config.PROXY_NTLM_DOMAIN, textProxyNtlmDomain.getText());
config.setValue(Config.PROCESS_KEEP_ACCOUNT_TEAM, buttonKeepAccountTeam.getSelection());
config.setValue(Config.CACHE_DESCRIBE_GLOBAL_RESULTS, buttonCacheDescribeGlobalResults.getSelection());
config.setValue(Config.BULK_API_ENABLED, buttonUseBulkApi.getSelection());
config.setValue(Config.BULK_API_SERIAL_MODE, buttonBulkApiSerialMode.getSelection());
config.setValue(Config.BULK_API_ZIP_CONTENT, buttonBulkApiZipContent.getSelection());
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/salesforce/dataloader/ui/AuthenticationRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ private void loginAsync(){
return;
}
}
if (controller.login() && controller.setEntityDescribes()) {
if (controller.login() && controller.getEntityDescribes() != null) {
messenger.accept(Labels.getString("SettingsPage.loginSuccessful"));
controller.saveConfig();
controller.updateLoaderWindowTitleAndCacheUserInfoForTheSession();
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/com/salesforce/dataloader/ui/SettingsPage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,7 @@ public IWizardPage getNextPage() {
}

public static boolean isNeeded(Controller controller) {
return (!controller.loginIfSessionExists() || controller.getEntityDescribes() == null || controller
.getEntityDescribes().isEmpty());
return (!controller.isLoggedIn());
}

private void authenticationCompleted(Boolean success){
Expand Down
1 change: 1 addition & 0 deletions src/main/resources/labels.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ AdvancedSettingsDialog.latestLoggingFile=Logging output file:
AdvancedSettingsDialog.loggingConfigFile=Logging configuration file:
AdvancedSettingsDialog.checkUploadDelimiterCheckbox=Specify delimiter(s) for upload operations.
AdvancedSettingsDialog.closeWizardOnFinish=Close UI Wizard dialog when an operation is completed.
AdvancedSettingsDialog.cacheDescribeGlobalResults=Cache Salesforce object and field information across operations

InsertWizard.windowTitle=Load Inserts
InsertWizard.confFirstLine=You have chosen to insert new records. Click Yes to begin.
Expand Down
6 changes: 0 additions & 6 deletions src/test/java/com/salesforce/dataloader/client/PartnerClientTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,17 +202,12 @@ public void testDisconnect() throws Exception {
@Test
public void testSetEntityDescribe() throws Exception{
PartnerClient client = new PartnerClient(getController());
assertTrue(client.setEntityDescribes());
assertNotNull(client.getDescribeGlobalResults());
assertEquals(client.getEntityTypes().getSobjects().length, client
.getDescribeGlobalResults().size());
}

@Test
public void testDescribeSObjects() throws Exception {
PartnerClient client = new PartnerClient(getController());
assertTrue(client.getEntityDescribeMap().isEmpty());
client.setEntityDescribes();

int numDescribes = 0;
for (String objectType : client.getDescribeGlobalResults().keySet()){
Expand All @@ -221,7 +216,6 @@ public void testDescribeSObjects() throws Exception {
numDescribes++;
assertNotNull(describeResult);
assertEquals(objectType, describeResult.getName());
assertEquals(numDescribes, client.getEntityDescribeMap().size());
} catch (Exception ex) {
if (ex.getMessage().contains("jsonNot")) {
System.out.println("PartnerClient.testDescribeSObjects: Unable to call describeSObject for " + objectType);
Expand Down
Loading