From 7db9ed45e0239ec88fca5cc18cd0ed253205d835 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Tue, 9 Jul 2024 10:50:41 -0400 Subject: [PATCH 1/6] fix(imports) possible fix for imports running out of memory ref: #29162 --- .../action/ImportContentletsAction.java | 110 ++++++++++++------ .../workflows/business/WorkflowCache.java | 11 +- .../workflows/business/WorkflowCacheImpl.java | 38 +++++- .../business/WorkflowFactoryImpl.java | 12 +- 4 files changed, 134 insertions(+), 37 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java index 5eaebb68b449..06fe5acc7118 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java @@ -1,6 +1,8 @@ package com.dotmarketing.portlets.contentlet.action; import com.dotcms.business.CloseDBIfOpened; +import com.dotcms.concurrent.DotConcurrentFactory; +import com.dotcms.concurrent.DotSubmitter; import com.dotcms.mock.request.MockAttributeRequest; import com.dotcms.mock.request.MockHeaderRequest; import com.dotcms.mock.request.MockSessionRequest; @@ -21,6 +23,7 @@ import com.dotmarketing.portlets.structure.model.Field; import com.dotmarketing.portlets.structure.model.Structure; import com.dotmarketing.util.AdminLogger; +import com.dotmarketing.util.Config; import com.dotmarketing.util.ImportUtil; import com.dotmarketing.util.Logger; import com.dotmarketing.util.UtilMethods; @@ -32,8 +35,10 @@ import com.liferay.util.FileUtil; import com.liferay.util.servlet.SessionMessages; import com.liferay.util.servlet.UploadPortletRequest; +import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -110,7 +115,7 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po //Validation UploadPortletRequest uploadReq = PortalUtil.getUploadPortletRequest(req); - byte[] bytes = FileUtil.getBytes(uploadReq.getFile("file")); + File file = uploadReq.getFile("file"); this.detectEncodeType(session, file); @@ -124,7 +129,7 @@ else if(importContentletsForm.getWorkflowActionId().isEmpty()){ SessionMessages.add(req, ERROR, "Workflow-action-type-required"); setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS); } - else if (bytes == null || bytes.length == 0) { + else if (file == null || file.length() < 100) { SessionMessages.add(req, ERROR, "message.contentlet.file.required"); setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS); } else { @@ -137,18 +142,18 @@ else if (bytes == null || bytes.length == 0) { setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS); return; } - + Reader reader = null; + CsvReader csvreader = null; try { - Reader reader = null; - CsvReader csvreader = null; + String[] csvHeaders = null; int languageCodeHeaderColumn = -1; int countryCodeHeaderColumn = -1; if (importContentletsForm.getLanguage() == -1) - reader = new InputStreamReader(new ByteArrayInputStream(bytes), Charset.forName("UTF-8")); + reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), Charset.forName("UTF-8"))); else - reader = new InputStreamReader(new ByteArrayInputStream(bytes)); + reader = new BufferedReader(new InputStreamReader(new FileInputStream(file))); csvreader = new CsvReader(reader); csvreader.setSafetySwitch(false); @@ -172,7 +177,7 @@ else if (bytes == null || bytes.length == 0) { SessionMessages.add(req, ERROR, "message.import.contentlet.csv_headers.required"); setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS); } else { - _generatePreview(0,req, res, config, form, user, bytes, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader); + _generatePreview(0,req, res, config, form, user, file, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader); setForward(req, "portlet.ext.contentlet.import_contentlets_preview"); } } else { @@ -189,15 +194,26 @@ else if (bytes == null || bytes.length == 0) { setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS); break; default: - _generatePreview(0, req, res, config, form, user, bytes, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader); + _generatePreview(0, req, res, config, form, user, file, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader); setForward(req, "portlet.ext.contentlet.import_contentlets_preview"); break; } - csvreader.close(); + } catch (Exception e) { _handleException(e, req); return; + }finally { + try { + csvreader.close(); + }catch (Exception e){ + Logger.error(this, e.getMessage()); + } + try{ + reader.close(); + }catch (Exception e){ + Logger.error(this, e.getMessage()); + } } } @@ -218,35 +234,40 @@ else if (bytes == null || bytes.length == 0) { final HttpServletRequest httpReq = reqImpl.getHttpServletRequest(); final HttpSession httpSession = httpReq.getSession(); final long importId = ImportAuditUtil.createAuditRecord(user.getUserId(), (String)httpSession.getAttribute("fileName")); - Thread t=new Thread() { + + + Runnable runnable = new Runnable() { + + @Override @CloseDBIfOpened public void run() { + Reader reader=null; + CsvReader csvreader=null; try { Logger.debug(this, "Calling Process File Method"); - - Reader reader; - CsvReader csvreader; + + String[] csvHeaders = null; int languageCodeHeaderColumn = -1; int countryCodeHeaderColumn = -1; - - byte[] bytes = (byte[]) httpSession.getAttribute("file_to_import"); + + ImportContentletsForm importContentletsForm = (ImportContentletsForm) form; String eCode = (String) httpSession.getAttribute(ENCODE_TYPE); if (importContentletsForm.getLanguage() == -1) { - reader = new InputStreamReader(new ByteArrayInputStream(bytes), - Charset.forName("UTF-8")); + reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")), + Charset.forName("UTF-8"))); } else if(eCode != null) { - reader = new InputStreamReader(new ByteArrayInputStream(bytes), - Charset.forName(eCode)); + reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")), + Charset.forName(eCode))); } else { - reader = new InputStreamReader(new ByteArrayInputStream(bytes)); + reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")))); } csvreader = new CsvReader(reader); csvreader.setSafetySwitch(false); - + if (importContentletsForm.getLanguage() == -1) { if (csvreader.readHeaders()) { csvHeaders = csvreader.getHeaders(); @@ -264,7 +285,7 @@ else if(eCode != null) { } } - final User user = _getUser(req); + HashMap> importresults= new HashMap<>(); if(importSession.equals(httpSession.getAttribute("importSession"))){ @@ -273,7 +294,7 @@ else if(eCode != null) { csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader,req); } - + final List counters= importresults.get("counters"); int contentsToImport=0; for(String counter: counters){ @@ -283,19 +304,34 @@ else if(eCode != null) { contentsToImport + Integer.parseInt(counterArray[1]); } } - + final List inodes= importresults.get("lastInode"); if(!inodes.isEmpty()){ ImportAuditUtil.updateAuditRecord(inodes.get(0), contentsToImport, importId,importresults); } - + csvreader.close(); - - + + } catch (Exception ae) { _handleException(ae, req); return; } finally{ + + try { + csvreader.close(); + }catch (Exception e){ + Logger.error(this, e.getMessage()); + } + try{ + reader.close(); + }catch (Exception e){ + Logger.error(this, e.getMessage()); + } + + + + if(!ImportAuditUtil.cancelledImports.containsKey(importId)){ ImportAuditUtil.setAuditRecordCompleted(importId); }else{ @@ -304,7 +340,15 @@ else if(eCode != null) { } } }; - t.start(); + + + if(Config.getBooleanProperty("IMPORT_CONTENTLETS_ASYNC", false)) { + runnable.run(); + }else{ + DotConcurrentFactory.getInstance().getSubmitter("importContentlets").submit(runnable); + } + + req.setAttribute("previewResults", session.getAttribute("previewResults")); session.removeAttribute("previewResults"); req.setAttribute("importId", importId); @@ -459,8 +503,8 @@ private void _downloadCSVTemplate(ActionRequest req, ActionResponse res, Portlet * the UI. * @param user * - The {@link User} performing this action. - * @param bytes - * - The byte array representation of the CSV file. + * @param file + * - The file representation of the CSV file. * @param csvHeaders * - The headers that make up the CSV file. * @param csvreader @@ -474,12 +518,12 @@ private void _downloadCSVTemplate(ActionRequest req, ActionResponse res, Portlet * @throws Exception * An error occurred when analyzing the CSV file. */ - private void _generatePreview(long importId, ActionRequest req, ActionResponse res, PortletConfig config, ActionForm form, User user, byte[] bytes, String[] csvHeaders, CsvReader csvreader, int languageCodeHeaderColumn, int countryCodeHeaderColumn, Reader reader) throws Exception { + private void _generatePreview(long importId, ActionRequest req, ActionResponse res, PortletConfig config, ActionForm form, User user, File file, String[] csvHeaders, CsvReader csvreader, int languageCodeHeaderColumn, int countryCodeHeaderColumn, Reader reader) throws Exception { // wraps request to get session object ActionRequestImpl reqImpl = (ActionRequestImpl) req; HttpServletRequest httpReq = reqImpl.getHttpServletRequest(); HttpSession session = httpReq.getSession(); - httpReq.getSession().setAttribute("file_to_import", bytes); + httpReq.getSession().setAttribute("file_to_import", file); httpReq.getSession().setAttribute("form_to_import", form); ImportContentletsForm importForm = (ImportContentletsForm) form; httpReq.getSession().setAttribute("fileName", importForm.getFileName()); diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java index 1385f3025f30..672ed292cf78 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; //This interface should have default package access public abstract class WorkflowCache implements Cachable { @@ -31,8 +32,16 @@ public abstract class WorkflowCache implements Cachable { abstract protected WorkflowStep add(WorkflowStep step); abstract protected List addActions(WorkflowStep step, List actions); abstract protected List addActions(WorkflowScheme scheme, List actions); + abstract protected List addActionClasses(WorkflowAction action, List actionClasses); abstract protected List getActions(WorkflowStep step); + + abstract protected Optional getAction(String actionId); + + abstract protected void addAction(WorkflowAction action); + + public abstract void removeAction(WorkflowAction action); + abstract protected List getActionClasses(final WorkflowAction action); abstract protected List getActions(WorkflowScheme scheme); @@ -165,4 +174,4 @@ public String[] getGroups() { * @param mapping SystemActionWorkflowActionMapping */ public abstract void removeSystemActionWorkflowActionMapping(SystemActionWorkflowActionMapping mapping); -} \ No newline at end of file +} diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java index ea6f8a02fc69..1f6e290b1478 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java @@ -17,6 +17,7 @@ import com.liferay.util.StringPool; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; //This interface should have default package access @@ -270,6 +271,7 @@ private String getKey (final String assetId, final long languageId) { protected List addActions(WorkflowStep step, List actions) { if(step == null || actions==null)return null; cache.put(step.getId(), actions, ACTION_GROUP); + actions.forEach(action -> addAction(action)); return actions; } @@ -280,6 +282,7 @@ protected List addActions(final WorkflowScheme scheme, if(scheme == null || actions==null)return null; cache.put(scheme.getId(), actions, ACTION_GROUP); + actions.forEach(action -> addAction(action)); return actions; } @@ -294,6 +297,39 @@ protected List getActions(WorkflowStep step) { return null; } + @Override + protected Optional getAction(String actionId) { + if(actionId == null ) { + return Optional.empty(); + } + try { + return Optional.ofNullable((WorkflowAction)cache.getNoThrow("action:" +actionId, ACTION_GROUP)); + } catch (Exception e) { + Logger.debug(WorkflowCacheImpl.class,e.getMessage(),e); + } + return Optional.empty(); + } + + @Override + public void addAction(WorkflowAction action) { + if(UtilMethods.isEmpty(()->action.getId())){ + return; + } + + cache.put("action:" +action.getId(), action, ACTION_GROUP); + } + + @Override + public void removeAction(WorkflowAction action) { + if(UtilMethods.isEmpty(()->action.getId())){ + return; + } + cache.remove("action:" +action.getId(), ACTION_GROUP); + } + + + + @Override protected List getActionClasses(final WorkflowAction action) { @@ -532,4 +568,4 @@ public void removeSystemActionWorkflowActionMapping(final SystemActionWorkflowAc } } } -} \ No newline at end of file +} diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java index a16675451ccd..73832d41f389 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java @@ -712,15 +712,21 @@ public void deleteWorkflowTask(WorkflowTask task) throws DotDataException { @Override public WorkflowAction findAction(String id) throws DotDataException { + Optional action = cache.getAction(id); + if(action.isPresent()) { + return action.get(); + } final DotConnect db = new DotConnect(); db.setSQL(WorkflowSQL.SELECT_ACTION); db.addParam(id); try { - return (WorkflowAction) this.convertListToObjects(db.loadObjectResults(), + WorkflowAction thisAction = (WorkflowAction) this.convertListToObjects(db.loadObjectResults(), WorkflowAction.class).get(0); + cache.addAction(thisAction); } catch (IndexOutOfBoundsException ioob) { - return null; + // no action found } + return null; } @Override @@ -1751,6 +1757,8 @@ public void saveAction(final WorkflowAction workflowAction, .loadResult(); } + cache.removeAction(workflowAction); + final WorkflowStep proxyStep = new WorkflowStep(); proxyStep.setId(workflowStep.getId()); cache.removeActions(proxyStep); From 446daec7fbd9ecad81b9da750cbed9fadc72fad7 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Tue, 9 Jul 2024 12:35:05 -0400 Subject: [PATCH 2/6] fix(imports) possible fix for imports running out of memory ref: #29162 --- .../contentlet/action/ImportContentletsAction.java | 8 ++++---- .../portlets/workflows/business/WorkflowFactoryImpl.java | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java index 06fe5acc7118..c19535dee5f2 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java @@ -341,11 +341,11 @@ else if(eCode != null) { } }; - - if(Config.getBooleanProperty("IMPORT_CONTENTLETS_ASYNC", false)) { - runnable.run(); - }else{ + + if(Config.getBooleanProperty("IMPORT_CONTENTLETS_ASYNC", true)) { DotConcurrentFactory.getInstance().getSubmitter("importContentlets").submit(runnable); + }else{ + runnable.run(); } diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java index 73832d41f389..ef69efa306f4 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java @@ -723,10 +723,11 @@ public WorkflowAction findAction(String id) throws DotDataException { WorkflowAction thisAction = (WorkflowAction) this.convertListToObjects(db.loadObjectResults(), WorkflowAction.class).get(0); cache.addAction(thisAction); + return thisAction; } catch (IndexOutOfBoundsException ioob) { - // no action found + return null; } - return null; + } @Override From ca82e174e4ed9b3776c8a8126f3fa3da53cddd5f Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Tue, 9 Jul 2024 14:32:32 -0400 Subject: [PATCH 3/6] fix(imports) fix cache invalidations ref: #29162 --- .../portlets/workflows/business/WorkflowFactoryImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java index ef69efa306f4..f43f3acda469 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java @@ -1861,7 +1861,7 @@ public void saveAction(final WorkflowAction action) final WorkflowScheme proxyScheme = new WorkflowScheme(); proxyScheme.setId(action.getSchemeId()); cache.removeActions(proxyScheme); - + cache.removeAction(action); // update workflowScheme mod date final WorkflowScheme scheme = findScheme(action.getSchemeId()); saveScheme(scheme); From 71a2d41440f008880a200eb2432ba83d5a900f50 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Tue, 9 Jul 2024 15:09:02 -0400 Subject: [PATCH 4/6] fix(imports) fix cache invalidations ref: #29162 --- .../action/ImportContentletsAction.java | 114 +++--------------- .../java/com/dotmarketing/util/FileUtil.java | 33 +++++ 2 files changed, 51 insertions(+), 96 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java index c19535dee5f2..0dbe7bf8da8a 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java @@ -2,7 +2,6 @@ import com.dotcms.business.CloseDBIfOpened; import com.dotcms.concurrent.DotConcurrentFactory; -import com.dotcms.concurrent.DotSubmitter; import com.dotcms.mock.request.MockAttributeRequest; import com.dotcms.mock.request.MockHeaderRequest; import com.dotcms.mock.request.MockSessionRequest; @@ -24,6 +23,7 @@ import com.dotmarketing.portlets.structure.model.Structure; import com.dotmarketing.util.AdminLogger; import com.dotmarketing.util.Config; +import com.dotmarketing.util.FileUtil; import com.dotmarketing.util.ImportUtil; import com.dotmarketing.util.Logger; import com.dotmarketing.util.UtilMethods; @@ -32,26 +32,20 @@ import com.liferay.portal.util.PortalUtil; import com.liferay.portlet.ActionRequestImpl; import com.liferay.portlet.ActionResponseImpl; -import com.liferay.util.FileUtil; import com.liferay.util.servlet.SessionMessages; import com.liferay.util.servlet.UploadPortletRequest; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.Charset; -import java.nio.file.Files; import java.util.HashMap; import java.util.List; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; -import org.mozilla.universalchardet.UniversalDetector; /** * This action class import content from csv/text files. The csv file should @@ -108,7 +102,7 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po /* * We are submiting the file to process */ - if ((cmd != null) && cmd.equals(com.dotmarketing.util.Constants.PREVIEW)) { + if (com.dotmarketing.util.Constants.PREVIEW.equals(cmd)) { try { Logger.debug(this, "Calling Preview Upload Method"); @@ -118,7 +112,7 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po File file = uploadReq.getFile("file"); - this.detectEncodeType(session, file); + final ImportContentletsForm importContentletsForm = (ImportContentletsForm) form; if(importContentletsForm.getStructure().isEmpty()){ @@ -142,20 +136,15 @@ else if (file == null || file.length() < 100) { setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS); return; } - Reader reader = null; - CsvReader csvreader = null; - try { + + Charset charset = importContentletsForm.getLanguage() == -1 ? Charset.defaultCharset() : FileUtil.detectEncodeType(file); + try(Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset))){ String[] csvHeaders = null; int languageCodeHeaderColumn = -1; int countryCodeHeaderColumn = -1; - - if (importContentletsForm.getLanguage() == -1) - reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), Charset.forName("UTF-8"))); - else - reader = new BufferedReader(new InputStreamReader(new FileInputStream(file))); - - csvreader = new CsvReader(reader); + + CsvReader csvreader = new CsvReader(reader); csvreader.setSafetySwitch(false); switch ((int) importContentletsForm.getLanguage()) { @@ -198,22 +187,7 @@ else if (file == null || file.length() < 100) { setForward(req, "portlet.ext.contentlet.import_contentlets_preview"); break; } - - } catch (Exception e) { - _handleException(e, req); - return; - }finally { - try { - csvreader.close(); - }catch (Exception e){ - Logger.error(this, e.getMessage()); - } - try{ - reader.close(); - }catch (Exception e){ - Logger.error(this, e.getMessage()); - } } } @@ -221,7 +195,7 @@ else if (file == null || file.length() < 100) { _handleException(ae, req); return; } - } else if ((cmd != null) && cmd.equals(com.dotmarketing.util.Constants.PUBLISH)) { + } else if (com.dotmarketing.util.Constants.PUBLISH.equals(cmd)) { AdminLogger.log(ImportContentletsAction.class, "processAction", "Importing Contentlets", user); Long importIdComplete=(Long)session.getAttribute("importId"); String subcmd = req.getParameter("subcmd"); @@ -241,9 +215,13 @@ else if (file == null || file.length() < 100) { @Override @CloseDBIfOpened public void run() { - Reader reader=null; - CsvReader csvreader=null; - try { + + ImportContentletsForm importContentletsForm = (ImportContentletsForm) form; + Charset charset = importContentletsForm.getLanguage() == -1 + ? Charset.defaultCharset() + : FileUtil.detectEncodeType((File) httpSession.getAttribute("file_to_import")); + try(Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")), charset))){ + Logger.debug(this, "Calling Process File Method"); @@ -252,20 +230,7 @@ public void run() { int countryCodeHeaderColumn = -1; - ImportContentletsForm importContentletsForm = (ImportContentletsForm) form; - String eCode = (String) httpSession.getAttribute(ENCODE_TYPE); - if (importContentletsForm.getLanguage() == -1) { - reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")), - Charset.forName("UTF-8"))); - } - else if(eCode != null) { - reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")), - Charset.forName(eCode))); - } - else { - reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")))); - } - csvreader = new CsvReader(reader); + CsvReader csvreader = new CsvReader(reader); csvreader.setSafetySwitch(false); if (importContentletsForm.getLanguage() == -1) { @@ -315,22 +280,8 @@ else if(eCode != null) { } catch (Exception ae) { _handleException(ae, req); - return; - } finally{ - - try { - csvreader.close(); - }catch (Exception e){ - Logger.error(this, e.getMessage()); - } - try{ - reader.close(); - }catch (Exception e){ - Logger.error(this, e.getMessage()); - } - - + } finally{ if(!ImportAuditUtil.cancelledImports.containsKey(importId)){ ImportAuditUtil.setAuditRecordCompleted(importId); @@ -381,37 +332,8 @@ else if(eCode != null) { } - /** - * - * @param session - * @param file - * @throws IOException - */ - private void detectEncodeType(final HttpSession session, final File file) throws IOException { - - String encodeType = null; - - if (null != file && file.exists()) { - byte[] buf = new byte[4096]; - try (InputStream is = Files.newInputStream(file.toPath())){ - UniversalDetector detector = new UniversalDetector(null); - int nread; - while ((nread = is.read(buf)) > 0 && !detector.isDone()) { - detector.handleData(buf, 0, nread); - } - detector.dataEnd(); - encodeType = detector.getDetectedCharset(); - session.setAttribute(ENCODE_TYPE, encodeType); - detector.reset(); - }catch (IOException e){ - Logger.error(this.getClass(), e.getMessage()); - throw e; - } - } - } - /** * Provides the user a CSV template based on the selected Content Type. This * assists users in the process of adding new content to dotCMS as it diff --git a/dotCMS/src/main/java/com/dotmarketing/util/FileUtil.java b/dotCMS/src/main/java/com/dotmarketing/util/FileUtil.java index 5d28e6686824..7eb7f040cec9 100644 --- a/dotCMS/src/main/java/com/dotmarketing/util/FileUtil.java +++ b/dotCMS/src/main/java/com/dotmarketing/util/FileUtil.java @@ -9,6 +9,7 @@ import com.liferay.util.StringPool; import io.vavr.Lazy; import io.vavr.control.Try; +import java.nio.charset.Charset; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; @@ -35,6 +36,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Stream; +import org.mozilla.universalchardet.UniversalDetector; /** * Provide utility methods to work with binary files in dotCMS. @@ -492,8 +494,39 @@ public static String getImageExtensionFromMIMEType(final String mimeType) { return StringPool.BLANK; } + + /** + * + * @param file + * @throws IOException + */ + public static Charset detectEncodeType(final File file) { + + byte[] buf = new byte[4096]; + try (InputStream is = Files.newInputStream(file.toPath())){ + + + UniversalDetector detector = new UniversalDetector(null); + int nread; + while ((nread = is.read(buf)) > 0 && !detector.isDone()) { + detector.handleData(buf, 0, nread); + } + detector.dataEnd(); + return Charset.forName(detector.getDetectedCharset()); + }catch (Exception e){ + Logger.error(FileUtil.class, e.getMessage(),e); + + } + return Charset.defaultCharset(); + + } + + + } + + final class PNGFileNameFilter implements FilenameFilter { public boolean accept(File dir, String name) { return (name.indexOf(".png") > -1); From 6845e2aa7533f0da15aeebf000d87f1352520fe9 Mon Sep 17 00:00:00 2001 From: erickgonzalez Date: Tue, 27 Aug 2024 09:51:44 -0600 Subject: [PATCH 5/6] revert: revert unrequired changes ref: #29162 --- .../workflows/business/WorkflowCache.java | 8 ----- .../workflows/business/WorkflowCacheImpl.java | 36 ------------------- .../business/WorkflowFactoryImpl.java | 12 +------ 3 files changed, 1 insertion(+), 55 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java index 672ed292cf78..37bc8e99e667 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java @@ -7,7 +7,6 @@ import java.util.List; import java.util.Map; -import java.util.Optional; //This interface should have default package access public abstract class WorkflowCache implements Cachable { @@ -32,16 +31,9 @@ public abstract class WorkflowCache implements Cachable { abstract protected WorkflowStep add(WorkflowStep step); abstract protected List addActions(WorkflowStep step, List actions); abstract protected List addActions(WorkflowScheme scheme, List actions); - abstract protected List addActionClasses(WorkflowAction action, List actionClasses); abstract protected List getActions(WorkflowStep step); - abstract protected Optional getAction(String actionId); - - abstract protected void addAction(WorkflowAction action); - - public abstract void removeAction(WorkflowAction action); - abstract protected List getActionClasses(final WorkflowAction action); abstract protected List getActions(WorkflowScheme scheme); diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java index 1f6e290b1478..5764d7311df5 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java @@ -17,7 +17,6 @@ import com.liferay.util.StringPool; import java.util.List; import java.util.Map; -import java.util.Optional; import org.apache.commons.lang3.StringUtils; //This interface should have default package access @@ -271,7 +270,6 @@ private String getKey (final String assetId, final long languageId) { protected List addActions(WorkflowStep step, List actions) { if(step == null || actions==null)return null; cache.put(step.getId(), actions, ACTION_GROUP); - actions.forEach(action -> addAction(action)); return actions; } @@ -282,7 +280,6 @@ protected List addActions(final WorkflowScheme scheme, if(scheme == null || actions==null)return null; cache.put(scheme.getId(), actions, ACTION_GROUP); - actions.forEach(action -> addAction(action)); return actions; } @@ -297,39 +294,6 @@ protected List getActions(WorkflowStep step) { return null; } - @Override - protected Optional getAction(String actionId) { - if(actionId == null ) { - return Optional.empty(); - } - try { - return Optional.ofNullable((WorkflowAction)cache.getNoThrow("action:" +actionId, ACTION_GROUP)); - } catch (Exception e) { - Logger.debug(WorkflowCacheImpl.class,e.getMessage(),e); - } - return Optional.empty(); - } - - @Override - public void addAction(WorkflowAction action) { - if(UtilMethods.isEmpty(()->action.getId())){ - return; - } - - cache.put("action:" +action.getId(), action, ACTION_GROUP); - } - - @Override - public void removeAction(WorkflowAction action) { - if(UtilMethods.isEmpty(()->action.getId())){ - return; - } - cache.remove("action:" +action.getId(), ACTION_GROUP); - } - - - - @Override protected List getActionClasses(final WorkflowAction action) { diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java index 35476e7bd5d8..2cf5850c04bf 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java @@ -713,22 +713,15 @@ public void deleteWorkflowTask(WorkflowTask task) throws DotDataException { @Override public WorkflowAction findAction(String id) throws DotDataException { - Optional action = cache.getAction(id); - if(action.isPresent()) { - return action.get(); - } final DotConnect db = new DotConnect(); db.setSQL(WorkflowSQL.SELECT_ACTION); db.addParam(id); try { - WorkflowAction thisAction = (WorkflowAction) this.convertListToObjects(db.loadObjectResults(), + return (WorkflowAction) this.convertListToObjects(db.loadObjectResults(), WorkflowAction.class).get(0); - cache.addAction(thisAction); - return thisAction; } catch (IndexOutOfBoundsException ioob) { return null; } - } @Override @@ -1759,8 +1752,6 @@ public void saveAction(final WorkflowAction workflowAction, .loadResult(); } - cache.removeAction(workflowAction); - final WorkflowStep proxyStep = new WorkflowStep(); proxyStep.setId(workflowStep.getId()); cache.removeActions(proxyStep); @@ -1862,7 +1853,6 @@ public void saveAction(final WorkflowAction action) final WorkflowScheme proxyScheme = new WorkflowScheme(); proxyScheme.setId(action.getSchemeId()); cache.removeActions(proxyScheme); - cache.removeAction(action); // update workflowScheme mod date final WorkflowScheme scheme = findScheme(action.getSchemeId()); saveScheme(scheme); From 54b599310070c17c1046252bc203a9091be40545 Mon Sep 17 00:00:00 2001 From: erickgonzalez Date: Thu, 29 Aug 2024 08:07:10 -0600 Subject: [PATCH 6/6] fix: commitGranularity to 100 and copy CSV to tmp dir. ref: #29162 --- .../action/ImportContentletsAction.java | 18 +++++++++--------- .../java/com/dotmarketing/util/ImportUtil.java | 17 ++++++++--------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java index 0dbe7bf8da8a..8a073d5ade57 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java @@ -21,12 +21,7 @@ import com.dotmarketing.portlets.contentlet.struts.ImportContentletsForm; import com.dotmarketing.portlets.structure.model.Field; import com.dotmarketing.portlets.structure.model.Structure; -import com.dotmarketing.util.AdminLogger; -import com.dotmarketing.util.Config; -import com.dotmarketing.util.FileUtil; -import com.dotmarketing.util.ImportUtil; -import com.dotmarketing.util.Logger; -import com.dotmarketing.util.UtilMethods; +import com.dotmarketing.util.*; import com.liferay.portal.model.User; import com.liferay.portal.util.Constants; import com.liferay.portal.util.PortalUtil; @@ -40,6 +35,8 @@ import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import javax.servlet.ServletOutputStream; @@ -112,6 +109,8 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po File file = uploadReq.getFile("file"); + Files.copy(file.toPath(), new File(ConfigUtils.getAssetTempPath()+java.io.File.separator+file.getName()).toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING); + file = Paths.get(ConfigUtils.getAssetTempPath()+java.io.File.separator+file.getName()).toFile(); final ImportContentletsForm importContentletsForm = (ImportContentletsForm) form; @@ -187,7 +186,7 @@ else if (file == null || file.length() < 100) { setForward(req, "portlet.ext.contentlet.import_contentlets_preview"); break; } - + csvreader.close(); } } @@ -217,10 +216,11 @@ else if (file == null || file.length() < 100) { public void run() { ImportContentletsForm importContentletsForm = (ImportContentletsForm) form; + File fileToImport = (File) httpSession.getAttribute("file_to_import"); Charset charset = importContentletsForm.getLanguage() == -1 ? Charset.defaultCharset() - : FileUtil.detectEncodeType((File) httpSession.getAttribute("file_to_import")); - try(Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")), charset))){ + : FileUtil.detectEncodeType(fileToImport); + try(Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(fileToImport), charset))){ Logger.debug(this, "Calling Process File Method"); diff --git a/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java b/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java index 48d3823f98b0..cd3ea121d7c7 100644 --- a/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java +++ b/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java @@ -100,7 +100,7 @@ public class ImportUtil { private final static String languageCodeHeader = "languageCode"; private final static String countryCodeHeader = "countryCode"; - private final static int commitGranularity = 500; + private final static int commitGranularity = 100; private final static int sleepTime = 200; public static final String[] IMP_DATE_FORMATS = new String[] { "d-MMM-yy", "MMM-yy", "MMMM-yy", "d-MMM", "dd-MMM-yyyy", @@ -219,7 +219,6 @@ public static HashMap> importFile(Long importId, String cur } lineNumber++; // Log preview/import status every 100 processed records - int loggingPoint = 100; //Reading the whole file if (headers.size() > 0) { if (!preview) { @@ -231,10 +230,6 @@ public static HashMap> importFile(Long importId, String cur break; } lineNumber++; - if (lineNumber % loggingPoint == 0) { - final String action = preview ? "previewed." : "imported."; - Logger.info(ImportUtil.class, String.format("-> %d entries have been %s", lineNumber, action)); - } csvLine = csvreader.getValues(); try { lines++; @@ -281,9 +276,13 @@ public static HashMap> importFile(Long importId, String cur errors++; } - if ( !preview && (lineNumber % commitGranularity == 0) ) { - HibernateUtil.closeAndCommitTransaction(); - HibernateUtil.startTransaction(); + if (lineNumber % commitGranularity == 0) { + final String action = preview ? "previewed." : "imported."; + Logger.info(ImportUtil.class, String.format("-> %d entries have been %s", lineNumber, action)); + if(!preview) { + HibernateUtil.closeAndCommitTransaction(); + HibernateUtil.startTransaction(); + } } } catch (final DotRuntimeException ex) { String errorMessage = getErrorMsgFromException(user, ex);