diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java index 5eaebb68b449..8a073d5ade57 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java @@ -1,6 +1,7 @@ package com.dotmarketing.portlets.contentlet.action; import com.dotcms.business.CloseDBIfOpened; +import com.dotcms.concurrent.DotConcurrentFactory; import com.dotcms.mock.request.MockAttributeRequest; import com.dotcms.mock.request.MockHeaderRequest; import com.dotcms.mock.request.MockSessionRequest; @@ -20,33 +21,28 @@ import com.dotmarketing.portlets.contentlet.struts.ImportContentletsForm; import com.dotmarketing.portlets.structure.model.Field; import com.dotmarketing.portlets.structure.model.Structure; -import com.dotmarketing.util.AdminLogger; -import com.dotmarketing.util.ImportUtil; -import com.dotmarketing.util.Logger; -import com.dotmarketing.util.UtilMethods; +import com.dotmarketing.util.*; import com.liferay.portal.model.User; import com.liferay.portal.util.Constants; import com.liferay.portal.util.PortalUtil; import com.liferay.portlet.ActionRequestImpl; import com.liferay.portlet.ActionResponseImpl; -import com.liferay.util.FileUtil; import com.liferay.util.servlet.SessionMessages; import com.liferay.util.servlet.UploadPortletRequest; -import java.io.ByteArrayInputStream; +import java.io.BufferedReader; import java.io.File; -import java.io.IOException; -import java.io.InputStream; +import java.io.FileInputStream; import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.Charset; import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; -import org.mozilla.universalchardet.UniversalDetector; /** * This action class import content from csv/text files. The csv file should @@ -103,17 +99,19 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po /* * We are submiting the file to process */ - if ((cmd != null) && cmd.equals(com.dotmarketing.util.Constants.PREVIEW)) { + if (com.dotmarketing.util.Constants.PREVIEW.equals(cmd)) { try { Logger.debug(this, "Calling Preview Upload Method"); //Validation UploadPortletRequest uploadReq = PortalUtil.getUploadPortletRequest(req); - byte[] bytes = FileUtil.getBytes(uploadReq.getFile("file")); + File file = uploadReq.getFile("file"); - this.detectEncodeType(session, file); + Files.copy(file.toPath(), new File(ConfigUtils.getAssetTempPath()+java.io.File.separator+file.getName()).toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING); + file = Paths.get(ConfigUtils.getAssetTempPath()+java.io.File.separator+file.getName()).toFile(); + final ImportContentletsForm importContentletsForm = (ImportContentletsForm) form; if(importContentletsForm.getStructure().isEmpty()){ @@ -124,7 +122,7 @@ else if(importContentletsForm.getWorkflowActionId().isEmpty()){ SessionMessages.add(req, ERROR, "Workflow-action-type-required"); setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS); } - else if (bytes == null || bytes.length == 0) { + else if (file == null || file.length() < 100) { SessionMessages.add(req, ERROR, "message.contentlet.file.required"); setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS); } else { @@ -138,19 +136,14 @@ else if (bytes == null || bytes.length == 0) { return; } - try { - Reader reader = null; - CsvReader csvreader = null; + Charset charset = importContentletsForm.getLanguage() == -1 ? Charset.defaultCharset() : FileUtil.detectEncodeType(file); + try(Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset))){ + String[] csvHeaders = null; int languageCodeHeaderColumn = -1; int countryCodeHeaderColumn = -1; - - if (importContentletsForm.getLanguage() == -1) - reader = new InputStreamReader(new ByteArrayInputStream(bytes), Charset.forName("UTF-8")); - else - reader = new InputStreamReader(new ByteArrayInputStream(bytes)); - - csvreader = new CsvReader(reader); + + CsvReader csvreader = new CsvReader(reader); csvreader.setSafetySwitch(false); switch ((int) importContentletsForm.getLanguage()) { @@ -172,7 +165,7 @@ else if (bytes == null || bytes.length == 0) { SessionMessages.add(req, ERROR, "message.import.contentlet.csv_headers.required"); setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS); } else { - _generatePreview(0,req, res, config, form, user, bytes, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader); + _generatePreview(0,req, res, config, form, user, file, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader); setForward(req, "portlet.ext.contentlet.import_contentlets_preview"); } } else { @@ -189,15 +182,11 @@ else if (bytes == null || bytes.length == 0) { setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS); break; default: - _generatePreview(0, req, res, config, form, user, bytes, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader); + _generatePreview(0, req, res, config, form, user, file, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader); setForward(req, "portlet.ext.contentlet.import_contentlets_preview"); break; } - csvreader.close(); - } catch (Exception e) { - _handleException(e, req); - return; } } @@ -205,7 +194,7 @@ else if (bytes == null || bytes.length == 0) { _handleException(ae, req); return; } - } else if ((cmd != null) && cmd.equals(com.dotmarketing.util.Constants.PUBLISH)) { + } else if (com.dotmarketing.util.Constants.PUBLISH.equals(cmd)) { AdminLogger.log(ImportContentletsAction.class, "processAction", "Importing Contentlets", user); Long importIdComplete=(Long)session.getAttribute("importId"); String subcmd = req.getParameter("subcmd"); @@ -218,35 +207,32 @@ else if (bytes == null || bytes.length == 0) { final HttpServletRequest httpReq = reqImpl.getHttpServletRequest(); final HttpSession httpSession = httpReq.getSession(); final long importId = ImportAuditUtil.createAuditRecord(user.getUserId(), (String)httpSession.getAttribute("fileName")); - Thread t=new Thread() { + + + Runnable runnable = new Runnable() { + + @Override @CloseDBIfOpened public void run() { - try { + + ImportContentletsForm importContentletsForm = (ImportContentletsForm) form; + File fileToImport = (File) httpSession.getAttribute("file_to_import"); + Charset charset = importContentletsForm.getLanguage() == -1 + ? Charset.defaultCharset() + : FileUtil.detectEncodeType(fileToImport); + try(Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(fileToImport), charset))){ + Logger.debug(this, "Calling Process File Method"); - - Reader reader; - CsvReader csvreader; + + String[] csvHeaders = null; int languageCodeHeaderColumn = -1; int countryCodeHeaderColumn = -1; - - byte[] bytes = (byte[]) httpSession.getAttribute("file_to_import"); - ImportContentletsForm importContentletsForm = (ImportContentletsForm) form; - String eCode = (String) httpSession.getAttribute(ENCODE_TYPE); - if (importContentletsForm.getLanguage() == -1) { - reader = new InputStreamReader(new ByteArrayInputStream(bytes), - Charset.forName("UTF-8")); - } - else if(eCode != null) { - reader = new InputStreamReader(new ByteArrayInputStream(bytes), - Charset.forName(eCode)); - } - else { - reader = new InputStreamReader(new ByteArrayInputStream(bytes)); - } - csvreader = new CsvReader(reader); + + + CsvReader csvreader = new CsvReader(reader); csvreader.setSafetySwitch(false); - + if (importContentletsForm.getLanguage() == -1) { if (csvreader.readHeaders()) { csvHeaders = csvreader.getHeaders(); @@ -264,7 +250,7 @@ else if(eCode != null) { } } - final User user = _getUser(req); + HashMap> importresults= new HashMap<>(); if(importSession.equals(httpSession.getAttribute("importSession"))){ @@ -273,7 +259,7 @@ else if(eCode != null) { csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader,req); } - + final List counters= importresults.get("counters"); int contentsToImport=0; for(String counter: counters){ @@ -283,19 +269,20 @@ else if(eCode != null) { contentsToImport + Integer.parseInt(counterArray[1]); } } - + final List inodes= importresults.get("lastInode"); if(!inodes.isEmpty()){ ImportAuditUtil.updateAuditRecord(inodes.get(0), contentsToImport, importId,importresults); } - + csvreader.close(); - - + + } catch (Exception ae) { _handleException(ae, req); - return; + } finally{ + if(!ImportAuditUtil.cancelledImports.containsKey(importId)){ ImportAuditUtil.setAuditRecordCompleted(importId); }else{ @@ -304,7 +291,15 @@ else if(eCode != null) { } } }; - t.start(); + + + if(Config.getBooleanProperty("IMPORT_CONTENTLETS_ASYNC", true)) { + DotConcurrentFactory.getInstance().getSubmitter("importContentlets").submit(runnable); + }else{ + runnable.run(); + } + + req.setAttribute("previewResults", session.getAttribute("previewResults")); session.removeAttribute("previewResults"); req.setAttribute("importId", importId); @@ -337,37 +332,8 @@ else if(eCode != null) { } - /** - * - * @param session - * @param file - * @throws IOException - */ - private void detectEncodeType(final HttpSession session, final File file) throws IOException { - - String encodeType = null; - - if (null != file && file.exists()) { - byte[] buf = new byte[4096]; - try (InputStream is = Files.newInputStream(file.toPath())){ - UniversalDetector detector = new UniversalDetector(null); - int nread; - while ((nread = is.read(buf)) > 0 && !detector.isDone()) { - detector.handleData(buf, 0, nread); - } - detector.dataEnd(); - encodeType = detector.getDetectedCharset(); - session.setAttribute(ENCODE_TYPE, encodeType); - detector.reset(); - }catch (IOException e){ - Logger.error(this.getClass(), e.getMessage()); - throw e; - } - } - } - /** * Provides the user a CSV template based on the selected Content Type. This * assists users in the process of adding new content to dotCMS as it @@ -459,8 +425,8 @@ private void _downloadCSVTemplate(ActionRequest req, ActionResponse res, Portlet * the UI. * @param user * - The {@link User} performing this action. - * @param bytes - * - The byte array representation of the CSV file. + * @param file + * - The file representation of the CSV file. * @param csvHeaders * - The headers that make up the CSV file. * @param csvreader @@ -474,12 +440,12 @@ private void _downloadCSVTemplate(ActionRequest req, ActionResponse res, Portlet * @throws Exception * An error occurred when analyzing the CSV file. */ - private void _generatePreview(long importId, ActionRequest req, ActionResponse res, PortletConfig config, ActionForm form, User user, byte[] bytes, String[] csvHeaders, CsvReader csvreader, int languageCodeHeaderColumn, int countryCodeHeaderColumn, Reader reader) throws Exception { + private void _generatePreview(long importId, ActionRequest req, ActionResponse res, PortletConfig config, ActionForm form, User user, File file, String[] csvHeaders, CsvReader csvreader, int languageCodeHeaderColumn, int countryCodeHeaderColumn, Reader reader) throws Exception { // wraps request to get session object ActionRequestImpl reqImpl = (ActionRequestImpl) req; HttpServletRequest httpReq = reqImpl.getHttpServletRequest(); HttpSession session = httpReq.getSession(); - httpReq.getSession().setAttribute("file_to_import", bytes); + httpReq.getSession().setAttribute("file_to_import", file); httpReq.getSession().setAttribute("form_to_import", form); ImportContentletsForm importForm = (ImportContentletsForm) form; httpReq.getSession().setAttribute("fileName", importForm.getFileName()); diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java index 1385f3025f30..37bc8e99e667 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCache.java @@ -33,6 +33,7 @@ public abstract class WorkflowCache implements Cachable { abstract protected List addActions(WorkflowScheme scheme, List actions); abstract protected List addActionClasses(WorkflowAction action, List actionClasses); abstract protected List getActions(WorkflowStep step); + abstract protected List getActionClasses(final WorkflowAction action); abstract protected List getActions(WorkflowScheme scheme); @@ -165,4 +166,4 @@ public String[] getGroups() { * @param mapping SystemActionWorkflowActionMapping */ public abstract void removeSystemActionWorkflowActionMapping(SystemActionWorkflowActionMapping mapping); -} \ No newline at end of file +} diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java index ea6f8a02fc69..5764d7311df5 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowCacheImpl.java @@ -532,4 +532,4 @@ public void removeSystemActionWorkflowActionMapping(final SystemActionWorkflowAc } } } -} \ No newline at end of file +} diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java index f9afdb98024e..2cf5850c04bf 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java @@ -717,7 +717,7 @@ public WorkflowAction findAction(String id) throws DotDataException { db.setSQL(WorkflowSQL.SELECT_ACTION); db.addParam(id); try { - return (WorkflowAction) this.convertListToObjects(db.loadObjectResults(), + return (WorkflowAction) this.convertListToObjects(db.loadObjectResults(), WorkflowAction.class).get(0); } catch (IndexOutOfBoundsException ioob) { return null; @@ -1853,7 +1853,6 @@ public void saveAction(final WorkflowAction action) final WorkflowScheme proxyScheme = new WorkflowScheme(); proxyScheme.setId(action.getSchemeId()); cache.removeActions(proxyScheme); - // update workflowScheme mod date final WorkflowScheme scheme = findScheme(action.getSchemeId()); saveScheme(scheme); diff --git a/dotCMS/src/main/java/com/dotmarketing/util/FileUtil.java b/dotCMS/src/main/java/com/dotmarketing/util/FileUtil.java index 5d28e6686824..7eb7f040cec9 100644 --- a/dotCMS/src/main/java/com/dotmarketing/util/FileUtil.java +++ b/dotCMS/src/main/java/com/dotmarketing/util/FileUtil.java @@ -9,6 +9,7 @@ import com.liferay.util.StringPool; import io.vavr.Lazy; import io.vavr.control.Try; +import java.nio.charset.Charset; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; @@ -35,6 +36,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Stream; +import org.mozilla.universalchardet.UniversalDetector; /** * Provide utility methods to work with binary files in dotCMS. @@ -492,8 +494,39 @@ public static String getImageExtensionFromMIMEType(final String mimeType) { return StringPool.BLANK; } + + /** + * + * @param file + * @throws IOException + */ + public static Charset detectEncodeType(final File file) { + + byte[] buf = new byte[4096]; + try (InputStream is = Files.newInputStream(file.toPath())){ + + + UniversalDetector detector = new UniversalDetector(null); + int nread; + while ((nread = is.read(buf)) > 0 && !detector.isDone()) { + detector.handleData(buf, 0, nread); + } + detector.dataEnd(); + return Charset.forName(detector.getDetectedCharset()); + }catch (Exception e){ + Logger.error(FileUtil.class, e.getMessage(),e); + + } + return Charset.defaultCharset(); + + } + + + } + + final class PNGFileNameFilter implements FilenameFilter { public boolean accept(File dir, String name) { return (name.indexOf(".png") > -1); diff --git a/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java b/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java index 48d3823f98b0..cd3ea121d7c7 100644 --- a/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java +++ b/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java @@ -100,7 +100,7 @@ public class ImportUtil { private final static String languageCodeHeader = "languageCode"; private final static String countryCodeHeader = "countryCode"; - private final static int commitGranularity = 500; + private final static int commitGranularity = 100; private final static int sleepTime = 200; public static final String[] IMP_DATE_FORMATS = new String[] { "d-MMM-yy", "MMM-yy", "MMMM-yy", "d-MMM", "dd-MMM-yyyy", @@ -219,7 +219,6 @@ public static HashMap> importFile(Long importId, String cur } lineNumber++; // Log preview/import status every 100 processed records - int loggingPoint = 100; //Reading the whole file if (headers.size() > 0) { if (!preview) { @@ -231,10 +230,6 @@ public static HashMap> importFile(Long importId, String cur break; } lineNumber++; - if (lineNumber % loggingPoint == 0) { - final String action = preview ? "previewed." : "imported."; - Logger.info(ImportUtil.class, String.format("-> %d entries have been %s", lineNumber, action)); - } csvLine = csvreader.getValues(); try { lines++; @@ -281,9 +276,13 @@ public static HashMap> importFile(Long importId, String cur errors++; } - if ( !preview && (lineNumber % commitGranularity == 0) ) { - HibernateUtil.closeAndCommitTransaction(); - HibernateUtil.startTransaction(); + if (lineNumber % commitGranularity == 0) { + final String action = preview ? "previewed." : "imported."; + Logger.info(ImportUtil.class, String.format("-> %d entries have been %s", lineNumber, action)); + if(!preview) { + HibernateUtil.closeAndCommitTransaction(); + HibernateUtil.startTransaction(); + } } } catch (final DotRuntimeException ex) { String errorMessage = getErrorMsgFromException(user, ex);