Skip to content

Commit

Permalink
fix(imports) fix cache invalidations
Browse files Browse the repository at this point in the history 
ref: #29162
  • Loading branch information
@wezell
wezell committed Jul 9, 2024
1 parent ca82e17 commit 71a2d41
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 96 deletions.
114 changes: 18 additions & 96 deletions ...MS/src/main/java/com/dotmarketing/portlets/contentlet/action/ImportContentletsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import com.dotcms.business.CloseDBIfOpened;
import com.dotcms.concurrent.DotConcurrentFactory;
import com.dotcms.concurrent.DotSubmitter;
import com.dotcms.mock.request.MockAttributeRequest;
import com.dotcms.mock.request.MockHeaderRequest;
import com.dotcms.mock.request.MockSessionRequest;
Expand All @@ -24,6 +23,7 @@
import com.dotmarketing.portlets.structure.model.Structure;
import com.dotmarketing.util.AdminLogger;
import com.dotmarketing.util.Config;
import com.dotmarketing.util.FileUtil;
import com.dotmarketing.util.ImportUtil;
import com.dotmarketing.util.Logger;
import com.dotmarketing.util.UtilMethods;
Expand All @@ -32,26 +32,20 @@
import com.liferay.portal.util.PortalUtil;
import com.liferay.portlet.ActionRequestImpl;
import com.liferay.portlet.ActionResponseImpl;
import com.liferay.util.FileUtil;
import com.liferay.util.servlet.SessionMessages;
import com.liferay.util.servlet.UploadPortletRequest;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.List;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import org.mozilla.universalchardet.UniversalDetector;

/**
* This action class import content from csv/text files. The csv file should
Expand Down Expand Up @@ -108,7 +102,7 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po
/*
* We are submiting the file to process
*/
if ((cmd != null) && cmd.equals(com.dotmarketing.util.Constants.PREVIEW)) {
if (com.dotmarketing.util.Constants.PREVIEW.equals(cmd)) {

try {
Logger.debug(this, "Calling Preview Upload Method");
Expand All @@ -118,7 +112,7 @@ public void processAction(ActionMapping mapping, final ActionForm form, final Po


File file = uploadReq.getFile("file");
this.detectEncodeType(session, file);


final ImportContentletsForm importContentletsForm = (ImportContentletsForm) form;
if(importContentletsForm.getStructure().isEmpty()){
Expand All @@ -142,20 +136,15 @@ else if (file == null || file.length() < 100) {
setForward(req, PORTLET_EXT_CONTENTLET_IMPORT_CONTENTLETS);
return;
}
Reader reader = null;
CsvReader csvreader = null;
try {

Charset charset = importContentletsForm.getLanguage() == -1 ? Charset.defaultCharset() : FileUtil.detectEncodeType(file);
try(Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset))){

String[] csvHeaders = null;
int languageCodeHeaderColumn = -1;
int countryCodeHeaderColumn = -1;

if (importContentletsForm.getLanguage() == -1)
reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), Charset.forName("UTF-8")));
else
reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)));

csvreader = new CsvReader(reader);

CsvReader csvreader = new CsvReader(reader);
csvreader.setSafetySwitch(false);

switch ((int) importContentletsForm.getLanguage()) {
Expand Down Expand Up @@ -198,30 +187,15 @@ else if (file == null || file.length() < 100) {
setForward(req, "portlet.ext.contentlet.import_contentlets_preview");
break;
}


} catch (Exception e) {
_handleException(e, req);
return;
}finally {
try {
csvreader.close();
}catch (Exception e){
Logger.error(this, e.getMessage());
}
try{
reader.close();
}catch (Exception e){
Logger.error(this, e.getMessage());
}
}
}

} catch (Exception ae) {
_handleException(ae, req);
return;
}
} else if ((cmd != null) && cmd.equals(com.dotmarketing.util.Constants.PUBLISH)) {
} else if (com.dotmarketing.util.Constants.PUBLISH.equals(cmd)) {
AdminLogger.log(ImportContentletsAction.class, "processAction", "Importing Contentlets", user);
Long importIdComplete=(Long)session.getAttribute("importId");
String subcmd = req.getParameter("subcmd");
Expand All @@ -241,9 +215,13 @@ else if (file == null || file.length() < 100) {
@Override
@CloseDBIfOpened
public void run() {
Reader reader=null;
CsvReader csvreader=null;
try {

ImportContentletsForm importContentletsForm = (ImportContentletsForm) form;
Charset charset = importContentletsForm.getLanguage() == -1
? Charset.defaultCharset()
: FileUtil.detectEncodeType((File) httpSession.getAttribute("file_to_import"));
try(Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")), charset))){

Logger.debug(this, "Calling Process File Method");


Expand All @@ -252,20 +230,7 @@ public void run() {
int countryCodeHeaderColumn = -1;


ImportContentletsForm importContentletsForm = (ImportContentletsForm) form;
String eCode = (String) httpSession.getAttribute(ENCODE_TYPE);
if (importContentletsForm.getLanguage() == -1) {
reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")),
Charset.forName("UTF-8")));
}
else if(eCode != null) {
reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import")),
Charset.forName(eCode)));
}
else {
reader = new BufferedReader(new InputStreamReader(new FileInputStream((File) httpSession.getAttribute("file_to_import"))));
}
csvreader = new CsvReader(reader);
CsvReader csvreader = new CsvReader(reader);
csvreader.setSafetySwitch(false);

if (importContentletsForm.getLanguage() == -1) {
Expand Down Expand Up @@ -315,22 +280,8 @@ else if(eCode != null) {

} catch (Exception ae) {
_handleException(ae, req);
return;
} finally{

try {
csvreader.close();
}catch (Exception e){
Logger.error(this, e.getMessage());
}
try{
reader.close();
}catch (Exception e){
Logger.error(this, e.getMessage());
}



} finally{

if(!ImportAuditUtil.cancelledImports.containsKey(importId)){
ImportAuditUtil.setAuditRecordCompleted(importId);
Expand Down Expand Up @@ -381,37 +332,8 @@ else if(eCode != null) {

}

/**
*
* @param session
* @param file
* @throws IOException
*/
private void detectEncodeType(final HttpSession session, final File file) throws IOException {

String encodeType = null;

if (null != file && file.exists()) {
byte[] buf = new byte[4096];
try (InputStream is = Files.newInputStream(file.toPath())){


UniversalDetector detector = new UniversalDetector(null);
int nread;
while ((nread = is.read(buf)) > 0 && !detector.isDone()) {
detector.handleData(buf, 0, nread);
}
detector.dataEnd();
encodeType = detector.getDetectedCharset();
session.setAttribute(ENCODE_TYPE, encodeType);
detector.reset();
}catch (IOException e){
Logger.error(this.getClass(), e.getMessage());
throw e;
}
}
}

/**
* Provides the user a CSV template based on the selected Content Type. This
* assists users in the process of adding new content to dotCMS as it
Expand Down
33 changes: 33 additions & 0 deletions dotCMS/src/main/java/com/dotmarketing/util/FileUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.liferay.util.StringPool;
import io.vavr.Lazy;
import io.vavr.control.Try;
import java.nio.charset.Charset;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;

Expand All @@ -35,6 +36,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;
import org.mozilla.universalchardet.UniversalDetector;

/**
* Provide utility methods to work with binary files in dotCMS.
Expand Down Expand Up @@ -492,8 +494,39 @@ public static String getImageExtensionFromMIMEType(final String mimeType) {
return StringPool.BLANK;
}


/**
*
* @param file
* @throws IOException
*/
public static Charset detectEncodeType(final File file) {

byte[] buf = new byte[4096];
try (InputStream is = Files.newInputStream(file.toPath())){


UniversalDetector detector = new UniversalDetector(null);
int nread;
while ((nread = is.read(buf)) > 0 && !detector.isDone()) {
detector.handleData(buf, 0, nread);
}
detector.dataEnd();
return Charset.forName(detector.getDetectedCharset());
}catch (Exception e){
Logger.error(FileUtil.class, e.getMessage(),e);

}
return Charset.defaultCharset();

}



}



final class PNGFileNameFilter implements FilenameFilter {
public boolean accept(File dir, String name) {
return (name.indexOf(".png") > -1);
Expand Down

0 comments on commit 71a2d41

Please sign in to comment.