Skip to content

Commit

Permalink
fix(imports) possible fix for imports running out of memory
Browse files Browse the repository at this point in the history 
ref: #29162
  • Loading branch information
@wezell
wezell committed Jul 9, 2024
1 parent 9087246 commit 7db9ed4
Show file tree
Hide file tree
Showing 4 changed files with 134 additions and 37 deletions.
110 changes: 77 additions & 33 deletions ...MS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package com.dotmarketing.portlets.contentlet.action;

import com.dotcms.business.CloseDBIfOpened;
import com.dotcms.concurrent.DotConcurrentFactory;
import com.dotcms.concurrent.DotSubmitter;
import com.dotcms.mock.request.MockAttributeRequest;
import com.dotcms.mock.request.MockHeaderRequest;
import com.dotcms.mock.request.MockSessionRequest;
Expand All @@ -21,6 +23,7 @@
import com.dotmarketing.portlets.structure.model.Field;
import com.dotmarketing.portlets.structure.model.Structure;
import com.dotmarketing.util.AdminLogger;
import com.dotmarketing.util.Config;
import com.dotmarketing.util.ImportUtil;
import com.dotmarketing.util.Logger;
import com.dotmarketing.util.UtilMethods;
Expand All @@ -32,8 +35,10 @@
import com.liferay.util.FileUtil;
import com.liferay.util.servlet.SessionMessages;
import com.liferay.util.servlet.UploadPortletRequest;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
Expand Down Expand Up @@ -110,7 +115,7 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po

//Validation
UploadPortletRequest uploadReq = PortalUtil.getUploadPortletRequest(req);
byte[] bytes = FileUtil.getBytes(uploadReq.getFile("file"));


File file = uploadReq.getFile("file");
this.detectEncodeType(session, file);
Expand All @@ -124,7 +129,7 @@ else if(importContentletsForm.getWorkflowActionId().isEmpty()){
SessionMessages.add(req, ERROR, "Workflow-action-type-required");
setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS);
}
else if (bytes == null || bytes.length == 0) {
else if (file == null || file.length() < 100) {
SessionMessages.add(req, ERROR, "message.contentlet.file.required");
setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS);
} else {
Expand All @@ -137,18 +142,18 @@ else if (bytes == null || bytes.length == 0) {
setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS);
return;
}

Reader reader = null;
CsvReader csvreader = null;
try {
Reader reader = null;
CsvReader csvreader = null;

String[] csvHeaders = null;
int languageCodeHeaderColumn = -1;
int countryCodeHeaderColumn = -1;

if (importContentletsForm.getLanguage() == -1)
reader = new InputStreamReader(new ByteArrayInputStream(bytes), Charset.forName("UTF-8"));
reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), Charset.forName("UTF-8")));
else
reader = new InputStreamReader(new ByteArrayInputStream(bytes));
reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)));

csvreader = new CsvReader(reader);
csvreader.setSafetySwitch(false);
Expand All @@ -172,7 +177,7 @@ else if (bytes == null || bytes.length == 0) {
SessionMessages.add(req, ERROR, "message.import.contentlet.csv_headers.required");
setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS);
} else {
_generatePreview(0,req, res, config, form, user, bytes, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader);
_generatePreview(0,req, res, config, form, user, file, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader);
setForward(req, "portlet.ext.contentlet.import_contentlets_preview");
}
} else {
Expand All @@ -189,15 +194,26 @@ else if (bytes == null || bytes.length == 0) {
setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS);
break;
default:
_generatePreview(0, req, res, config, form, user, bytes, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader);
_generatePreview(0, req, res, config, form, user, file, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader);
setForward(req, "portlet.ext.contentlet.import_contentlets_preview");
break;
}

csvreader.close();

} catch (Exception e) {
_handleException(e, req);
return;
}finally {
try {
csvreader.close();
}catch (Exception e){
Logger.error(this, e.getMessage());
}
try{
reader.close();
}catch (Exception e){
Logger.error(this, e.getMessage());
}
}
}

Expand All @@ -218,35 +234,40 @@ else if (bytes == null || bytes.length == 0) {
final HttpServletRequest httpReq = reqImpl.getHttpServletRequest();
final HttpSession httpSession = httpReq.getSession();
final long importId = ImportAuditUtil.createAuditRecord(user.getUserId(), (String)httpSession.getAttribute("fileName"));
Thread t=new Thread() {


Runnable runnable = new Runnable() {

@Override
@CloseDBIfOpened
public void run() {
Reader reader=null;
CsvReader csvreader=null;
try {
Logger.debug(this, "Calling Process File Method");

Reader reader;
CsvReader csvreader;


String[] csvHeaders = null;
int languageCodeHeaderColumn = -1;
int countryCodeHeaderColumn = -1;
byte[] bytes = (byte[]) httpSession.getAttribute("file_to_import");


ImportContentletsForm importContentletsForm = (ImportContentletsForm) form;
String eCode = (String) httpSession.getAttribute(ENCODE_TYPE);
if (importContentletsForm.getLanguage() == -1) {
reader = new InputStreamReader(new ByteArrayInputStream(bytes),
Charset.forName("UTF-8"));
reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")),
Charset.forName("UTF-8")));
}
else if(eCode != null) {
reader = new InputStreamReader(new ByteArrayInputStream(bytes),
Charset.forName(eCode));
reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")),
Charset.forName(eCode)));
}
else {
reader = new InputStreamReader(new ByteArrayInputStream(bytes));
reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import"))));
}
csvreader = new CsvReader(reader);
csvreader.setSafetySwitch(false);

if (importContentletsForm.getLanguage() == -1) {
if (csvreader.readHeaders()) {
csvHeaders = csvreader.getHeaders();
Expand All @@ -264,7 +285,7 @@ else if(eCode != null) {
}
}

final User user = _getUser(req);


HashMap<String, List<String>> importresults= new HashMap<>();
if(importSession.equals(httpSession.getAttribute("importSession"))){
Expand All @@ -273,7 +294,7 @@ else if(eCode != null) {
csvHeaders, csvreader, languageCodeHeaderColumn,
countryCodeHeaderColumn, reader,req);
}

final List<String> counters= importresults.get("counters");
int contentsToImport=0;
for(String counter: counters){
Expand All @@ -283,19 +304,34 @@ else if(eCode != null) {
contentsToImport + Integer.parseInt(counterArray[1]);
}
}

final List<String> inodes= importresults.get("lastInode");
if(!inodes.isEmpty()){
ImportAuditUtil.updateAuditRecord(inodes.get(0), contentsToImport, importId,importresults);
}

csvreader.close();


} catch (Exception ae) {
_handleException(ae, req);
return;
} finally{

try {
csvreader.close();
}catch (Exception e){
Logger.error(this, e.getMessage());
}
try{
reader.close();
}catch (Exception e){
Logger.error(this, e.getMessage());
}




if(!ImportAuditUtil.cancelledImports.containsKey(importId)){
ImportAuditUtil.setAuditRecordCompleted(importId);
}else{
Expand All @@ -304,7 +340,15 @@ else if(eCode != null) {
}
}
};
t.start();


if(Config.getBooleanProperty("IMPORT_CONTENTLETS_ASYNC", false)) {
runnable.run();
}else{
DotConcurrentFactory.getInstance().getSubmitter("importContentlets").submit(runnable);
}


req.setAttribute("previewResults", session.getAttribute("previewResults"));
session.removeAttribute("previewResults");
req.setAttribute("importId", importId);
Expand Down Expand Up @@ -459,8 +503,8 @@ private void _downloadCSVTemplate(ActionRequest req, ActionResponse res, Portlet
* the UI.
* @param user
* - The {@link User} performing this action.
* @param bytes
* - The byte array representation of the CSV file.
* @param file
* - The file representation of the CSV file.
* @param csvHeaders
* - The headers that make up the CSV file.
* @param csvreader
Expand All @@ -474,12 +518,12 @@ private void _downloadCSVTemplate(ActionRequest req, ActionResponse res, Portlet
* @throws Exception
* An error occurred when analyzing the CSV file.
*/
private void _generatePreview(long importId, ActionRequest req, ActionResponse res, PortletConfig config, ActionForm form, User user, byte[] bytes, String[] csvHeaders, CsvReader csvreader, int languageCodeHeaderColumn, int countryCodeHeaderColumn, Reader reader) throws Exception {
private void _generatePreview(long importId, ActionRequest req, ActionResponse res, PortletConfig config, ActionForm form, User user, File file, String[] csvHeaders, CsvReader csvreader, int languageCodeHeaderColumn, int countryCodeHeaderColumn, Reader reader) throws Exception {
// wraps request to get session object
ActionRequestImpl reqImpl = (ActionRequestImpl) req;
HttpServletRequest httpReq = reqImpl.getHttpServletRequest();
HttpSession session = httpReq.getSession();
httpReq.getSession().setAttribute("file_to_import", bytes);
httpReq.getSession().setAttribute("file_to_import", file);
httpReq.getSession().setAttribute("form_to_import", form);
ImportContentletsForm importForm = (ImportContentletsForm) form;
httpReq.getSession().setAttribute("fileName", importForm.getFileName());
Expand Down
11 changes: 10 additions & 1 deletion dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import java.util.List;
import java.util.Map;
import java.util.Optional;

//This interface should have default package access
public abstract class WorkflowCache implements Cachable {
Expand All @@ -31,8 +32,16 @@ public abstract class WorkflowCache implements Cachable {
abstract protected WorkflowStep add(WorkflowStep step);
abstract protected List<WorkflowAction> addActions(WorkflowStep step, List<WorkflowAction> actions);
abstract protected List<WorkflowAction> addActions(WorkflowScheme scheme, List<WorkflowAction> actions);

abstract protected List<WorkflowActionClass> addActionClasses(WorkflowAction action, List<WorkflowActionClass> actionClasses);
abstract protected List<WorkflowAction> getActions(WorkflowStep step);

abstract protected Optional<WorkflowAction> getAction(String actionId);

abstract protected void addAction(WorkflowAction action);

public abstract void removeAction(WorkflowAction action);

abstract protected List<WorkflowActionClass> getActionClasses(final WorkflowAction action);
abstract protected List<WorkflowAction> getActions(WorkflowScheme scheme);

Expand Down Expand Up @@ -165,4 +174,4 @@ public String[] getGroups() {
* @param mapping SystemActionWorkflowActionMapping
*/
public abstract void removeSystemActionWorkflowActionMapping(SystemActionWorkflowActionMapping mapping);
}
}
38 changes: 37 additions & 1 deletion dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import com.liferay.util.StringPool;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;

//This interface should have default package access
Expand Down Expand Up @@ -270,6 +271,7 @@ private String getKey (final String assetId, final long languageId) {
protected List<WorkflowAction> addActions(WorkflowStep step, List<WorkflowAction> actions) {
if(step == null || actions==null)return null;
cache.put(step.getId(), actions, ACTION_GROUP);
actions.forEach(action -> addAction(action));
return actions;

}
Expand All @@ -280,6 +282,7 @@ protected List<WorkflowAction> addActions(final WorkflowScheme scheme,
if(scheme == null || actions==null)return null;

cache.put(scheme.getId(), actions, ACTION_GROUP);
actions.forEach(action -> addAction(action));
return actions;
}

Expand All @@ -294,6 +297,39 @@ protected List<WorkflowAction> getActions(WorkflowStep step) {
return null;
}

@Override
protected Optional<WorkflowAction> getAction(String actionId) {
if(actionId == null ) {
return Optional.empty();
}
try {
return Optional.ofNullable((WorkflowAction)cache.getNoThrow("action:" +actionId, ACTION_GROUP));
} catch (Exception e) {
Logger.debug(WorkflowCacheImpl.class,e.getMessage(),e);
}
return Optional.empty();
}

@Override
public void addAction(WorkflowAction action) {
if(UtilMethods.isEmpty(()->action.getId())){
return;
}

cache.put("action:" +action.getId(), action, ACTION_GROUP);
}

@Override
public void removeAction(WorkflowAction action) {
if(UtilMethods.isEmpty(()->action.getId())){
return;
}
cache.remove("action:" +action.getId(), ACTION_GROUP);
}




@Override
protected List<WorkflowActionClass> getActionClasses(final WorkflowAction action) {

Expand Down Expand Up @@ -532,4 +568,4 @@ public void removeSystemActionWorkflowActionMapping(final SystemActionWorkflowAc
}
}
}
}
}
12 changes: 10 additions & 2 deletions dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -712,15 +712,21 @@ public void deleteWorkflowTask(WorkflowTask task) throws DotDataException {

@Override
public WorkflowAction findAction(String id) throws DotDataException {
Optional<WorkflowAction> action = cache.getAction(id);
if(action.isPresent()) {
return action.get();
}
final DotConnect db = new DotConnect();
db.setSQL(WorkflowSQL.SELECT_ACTION);
db.addParam(id);
try {
return (WorkflowAction) this.convertListToObjects(db.loadObjectResults(),
WorkflowAction thisAction = (WorkflowAction) this.convertListToObjects(db.loadObjectResults(),
WorkflowAction.class).get(0);
cache.addAction(thisAction);
} catch (IndexOutOfBoundsException ioob) {
return null;
// no action found
}
return null;
}

@Override
Expand Down Expand Up @@ -1751,6 +1757,8 @@ public void saveAction(final WorkflowAction workflowAction,
.loadResult();
}

cache.removeAction(workflowAction);

final WorkflowStep proxyStep = new WorkflowStep();
proxyStep.setId(workflowStep.getId());
cache.removeActions(proxyStep);
Expand Down

0 comments on commit 7db9ed4

Please sign in to comment.