Skip to content

Commit

Permalink
#29162 include in 23.10.24 LTS
Browse files Browse the repository at this point in the history
  • Loading branch information
@erickgonzalez
erickgonzalez committed Sep 4, 2024
1 parent 5f9a3f8 commit b7fbe51
Show file tree
Hide file tree
Showing 4 changed files with 77 additions and 67 deletions.
3 changes: 2 additions & 1 deletion dotCMS/hotfix_tracking.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,4 +146,5 @@ This maintenance release includes the following code fixes:
139. https://github.com/dotCMS/core/issues/29254 : Add a new main tag for LTSs #29254
140. https://github.com/dotCMS/core/issues/28857 : dotAsset is Breaking FileViewStrategy #28857
141. https://github.com/dotCMS/core/issues/29259 : Page API: Image field content gone when request page with depth #29259
142. https://github.com/dotCMS/core/issues/29667 : Update CLEANUP_BUNDLES_OLDER_THAN_DAYS default value to 4 #29667
142. https://github.com/dotCMS/core/issues/29667 : Update CLEANUP_BUNDLES_OLDER_THAN_DAYS default value to 4 #29667
143. https://github.com/dotCMS/core/issues/29162 : Slow performance with imports in some cases #29162
94 changes: 39 additions & 55 deletions ...MS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.dotmarketing.portlets.contentlet.action;

import com.dotcms.business.CloseDBIfOpened;
import com.dotcms.concurrent.DotConcurrentFactory;
import com.dotcms.mock.request.MockAttributeRequest;
import com.dotcms.mock.request.MockHeaderRequest;
import com.dotcms.mock.request.MockSessionRequest;
Expand All @@ -20,10 +21,7 @@
import com.dotmarketing.portlets.contentlet.struts.ImportContentletsForm;
import com.dotmarketing.portlets.structure.model.Field;
import com.dotmarketing.portlets.structure.model.Structure;
import com.dotmarketing.util.AdminLogger;
import com.dotmarketing.util.ImportUtil;
import com.dotmarketing.util.Logger;
import com.dotmarketing.util.UtilMethods;
import com.dotmarketing.util.*;
import com.liferay.portal.model.User;
import com.liferay.portal.util.Constants;
import com.liferay.portal.util.PortalUtil;
Expand All @@ -32,14 +30,11 @@
import com.liferay.util.FileUtil;
import com.liferay.util.servlet.SessionMessages;
import com.liferay.util.servlet.UploadPortletRequest;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;

import java.io.*;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import javax.servlet.ServletOutputStream;
Expand Down Expand Up @@ -111,13 +106,15 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po
byte[] bytes = FileUtil.getBytes(uploadReq.getFile("file"));

File file = uploadReq.getFile("file");
this.detectEncodeType(session, file);
Files.copy(file.toPath(), new File(ConfigUtils.getAssetTempPath()+java.io.File.separator+file.getName()).toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING);
file = Paths.get(ConfigUtils.getAssetTempPath()+java.io.File.separator+file.getName()).toFile();

final ImportContentletsForm importContentletsForm = (ImportContentletsForm) form;
if(importContentletsForm.getStructure().isEmpty()){
SessionMessages.add(req, "error", "structure-type-is-required");
setForward(req, "portlet.ext.contentlet.import_contentlets");
}else if (bytes == null || bytes.length == 0) {
}
else if (file == null || file.length() < 100) {
SessionMessages.add(req, "error", "message.contentlet.file.required");
setForward(req, "portlet.ext.contentlet.import_contentlets");
} else {
Expand All @@ -131,19 +128,13 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po
return;
}

try {
Reader reader = null;
CsvReader csvreader = null;
Charset charset = importContentletsForm.getLanguage() == -1 ? Charset.defaultCharset() : FileUtil.detectEncodeType(file);
try(Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset))){
String[] csvHeaders = null;
int languageCodeHeaderColumn = -1;
int countryCodeHeaderColumn = -1;

if (importContentletsForm.getLanguage() == -1)
reader = new InputStreamReader(new ByteArrayInputStream(bytes), Charset.forName("UTF-8"));
else
reader = new InputStreamReader(new ByteArrayInputStream(bytes));

csvreader = new CsvReader(reader);

CsvReader csvreader = new CsvReader(reader);
csvreader.setSafetySwitch(false);

switch ((int) importContentletsForm.getLanguage()) {
Expand All @@ -165,7 +156,7 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po
SessionMessages.add(req, "error", "message.import.contentlet.csv_headers.required");
setForward(req, "portlet.ext.contentlet.import_contentlets");
} else {
_generatePreview(0,req, res, config, form, user, bytes, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader);
_generatePreview(0,req, res, config, form, user, file, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader);
setForward(req, "portlet.ext.contentlet.import_contentlets_preview");
}
} else {
Expand All @@ -182,7 +173,7 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po
setForward(req, "portlet.ext.contentlet.import_contentlets");
break;
default:
_generatePreview(0, req, res, config, form, user, bytes, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader);
_generatePreview(0,req, res, config, form, user, file, csvHeaders, csvreader, languageCodeHeaderColumn, countryCodeHeaderColumn, reader);
setForward(req, "portlet.ext.contentlet.import_contentlets_preview");
break;
}
Expand Down Expand Up @@ -211,33 +202,25 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po
final HttpServletRequest httpReq = reqImpl.getHttpServletRequest();
final HttpSession httpSession = httpReq.getSession();
final long importId = ImportAuditUtil.createAuditRecord(user.getUserId(), (String)httpSession.getAttribute("fileName"));
Thread t=new Thread() {
@CloseDBIfOpened
public void run() {
try {
Runnable runnable = new Runnable() {

@Override
@CloseDBIfOpened
public void run() {

ImportContentletsForm importContentletsForm = (ImportContentletsForm) form;
File fileToImport = (File) httpSession.getAttribute("file_to_import");
Charset charset = importContentletsForm.getLanguage() == -1
? Charset.defaultCharset()
: FileUtil.detectEncodeType(fileToImport);
try(Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(fileToImport), charset))){
Logger.debug(this, "Calling Process File Method");

Reader reader;
CsvReader csvreader;

String[] csvHeaders = null;
int languageCodeHeaderColumn = -1;
int countryCodeHeaderColumn = -1;

byte[] bytes = (byte[]) httpSession.getAttribute("file_to_import");
ImportContentletsForm importContentletsForm = (ImportContentletsForm) form;
String eCode = (String) httpSession.getAttribute(ENCODE_TYPE);
if (importContentletsForm.getLanguage() == -1) {
reader = new InputStreamReader(new ByteArrayInputStream(bytes),
Charset.forName("UTF-8"));
}
else if(eCode != null) {
reader = new InputStreamReader(new ByteArrayInputStream(bytes),
Charset.forName(eCode));
}
else {
reader = new InputStreamReader(new ByteArrayInputStream(bytes));
}
csvreader = new CsvReader(reader);

CsvReader csvreader = new CsvReader(reader);
csvreader.setSafetySwitch(false);

if (importContentletsForm.getLanguage() == -1) {
Expand All @@ -257,8 +240,6 @@ else if(eCode != null) {
}
}

final User user = _getUser(req);

HashMap<String, List<String>> importresults= new HashMap<>();
if(importSession.equals(httpSession.getAttribute("importSession"))){
httpSession.removeAttribute("importSession");
Expand Down Expand Up @@ -287,7 +268,6 @@ else if(eCode != null) {

} catch (Exception ae) {
_handleException(ae, req);
return;
} finally{
if(!ImportAuditUtil.cancelledImports.containsKey(importId)){
ImportAuditUtil.setAuditRecordCompleted(importId);
Expand All @@ -297,7 +277,11 @@ else if(eCode != null) {
}
}
};
t.start();
if(Config.getBooleanProperty("IMPORT_CONTENTLETS_ASYNC", true)) {
DotConcurrentFactory.getInstance().getSubmitter("importContentlets").submit(runnable);
}else{
runnable.run();
}
req.setAttribute("previewResults", session.getAttribute("previewResults"));
session.removeAttribute("previewResults");
req.setAttribute("importId", importId);
Expand Down Expand Up @@ -452,8 +436,8 @@ private void _downloadCSVTemplate(ActionRequest req, ActionResponse res, Portlet
* the UI.
* @param user
* - The {@link User} performing this action.
* @param bytes
* - The byte array representation of the CSV file.
* @param file
* - The file representation of the CSV file.
* @param csvHeaders
* - The headers that make up the CSV file.
* @param csvreader
Expand All @@ -467,12 +451,12 @@ private void _downloadCSVTemplate(ActionRequest req, ActionResponse res, Portlet
* @throws Exception
* An error occurred when analyzing the CSV file.
*/
private void _generatePreview(long importId, ActionRequest req, ActionResponse res, PortletConfig config, ActionForm form, User user, byte[] bytes, String[] csvHeaders, CsvReader csvreader, int languageCodeHeaderColumn, int countryCodeHeaderColumn, Reader reader) throws Exception {
private void _generatePreview(long importId, ActionRequest req, ActionResponse res, PortletConfig config, ActionForm form, User user, File file, String[] csvHeaders, CsvReader csvreader, int languageCodeHeaderColumn, int countryCodeHeaderColumn, Reader reader) throws Exception {
// wraps request to get session object
ActionRequestImpl reqImpl = (ActionRequestImpl) req;
HttpServletRequest httpReq = reqImpl.getHttpServletRequest();
HttpSession session = httpReq.getSession();
httpReq.getSession().setAttribute("file_to_import", bytes);
httpReq.getSession().setAttribute("file_to_import", file);
httpReq.getSession().setAttribute("form_to_import", form);
ImportContentletsForm importForm = (ImportContentletsForm) form;
httpReq.getSession().setAttribute("fileName", importForm.getFileName());
Expand Down
19 changes: 8 additions & 11 deletions dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,7 @@ public class ImportUtil {
private final static String languageCodeHeader = "languageCode";
private final static String countryCodeHeader = "countryCode";

private final static int commitGranularity = 500;
private final static int sleepTime = 200;
private final static int commitGranularity = 100;

public static final String[] IMP_DATE_FORMATS = new String[] { "d-MMM-yy", "MMM-yy", "MMMM-yy", "d-MMM", "dd-MMM-yyyy",
"MM/dd/yy hh:mm aa", "MM/dd/yyyy hh:mm aa", "MM/dd/yy HH:mm", "MM/dd/yyyy HH:mm", "MMMM dd, yyyy", "M/d/y", "M/d",
Expand Down Expand Up @@ -223,8 +222,6 @@ public static HashMap<String, List<String>> importFile(Long importId, String cur
importHeaders(csvreader.getHeaders(), contentType, keyfields, isMultilingual, user, results, headers, keyFields, uniqueFields,relationships,onlyChild,onlyParent);
}
lineNumber++;
// Log preview/import status every 100 processed records
int loggingPoint = 100;
//Reading the whole file
if (headers.size() > 0) {
if (!preview) {
Expand All @@ -236,10 +233,6 @@ public static HashMap<String, List<String>> importFile(Long importId, String cur
break;
}
lineNumber++;
if (lineNumber % loggingPoint == 0) {
final String action = preview ? "previewed." : "imported.";
Logger.info(ImportUtil.class, String.format("-> %d entries have been %s", lineNumber, action));
}
csvLine = csvreader.getValues();
try {
lines++;
Expand Down Expand Up @@ -286,9 +279,13 @@ public static HashMap<String, List<String>> importFile(Long importId, String cur
errors++;
}

if ( !preview && (lineNumber % commitGranularity == 0) ) {
HibernateUtil.closeAndCommitTransaction();
HibernateUtil.startTransaction();
if (lineNumber % commitGranularity == 0) {
final String action = preview ? "previewed." : "imported.";
Logger.info(ImportUtil.class, String.format("-> %d entries have been %s", lineNumber, action));
if (!preview) {
HibernateUtil.closeAndCommitTransaction();
HibernateUtil.startTransaction();
}
}
} catch (final DotRuntimeException ex) {
String errorMessage = getErrorMsgFromException(user, ex);
Expand Down
28 changes: 28 additions & 0 deletions dotCMS/src/main/java/com/liferay/util/FileUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
package com.liferay.util;

import com.dotcms.publisher.pusher.PushUtils;

import java.nio.charset.Charset;
import java.util.Comparator;
import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
Expand Down Expand Up @@ -66,6 +68,7 @@
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
import org.mozilla.universalchardet.UniversalDetector;

/**
* <a href="FileUtil.java.html"><b><i>View Source</i></b></a>
Expand Down Expand Up @@ -1083,6 +1086,31 @@ public static File tarGzipDirectory(final File directory) throws IOException {
}


/**
*
* @param file
* @throws IOException
*/
public static Charset detectEncodeType(final File file) {

byte[] buf = new byte[4096];
try (InputStream is = Files.newInputStream(file.toPath())){


UniversalDetector detector = new UniversalDetector(null);
int nread;
while ((nread = is.read(buf)) > 0 && !detector.isDone()) {
detector.handleData(buf, 0, nread);
}
detector.dataEnd();
return Charset.forName(detector.getDetectedCharset());
}catch (Exception e){
Logger.error(FileUtil.class, e.getMessage(),e);

}
return Charset.defaultCharset();

}


}

0 comments on commit b7fbe51

Please sign in to comment.