From 21e42d3d00bb98daa36620bb589ea512ac9c124b Mon Sep 17 00:00:00 2001 From: lbownik Date: Fri, 15 Nov 2024 09:41:25 +0100 Subject: [PATCH] =?UTF-8?q?Fixed=20"Failed=20to=20delete=20temp=20like"=20?= =?UTF-8?q?in=20IngestableDataChecker=20under=20Win=E2=80=A6=20(#2584)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed "Failed to delete temp file" in IngestableDataChecker under Windows. --- .../ingest/IngestableDataChecker.java | 45 +++-- .../ingest/IngestableDataCheckerTest.java | 175 ++++-------------- 2 files changed, 54 insertions(+), 166 deletions(-) diff --git a/dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java b/dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java index 3a88117b90..fb80f855f2 100644 --- a/dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java +++ b/dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java @@ -20,9 +20,10 @@ package edu.harvard.iq.dataverse.ingest; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.builder.ToStringBuilder; -import org.apache.commons.lang.builder.ToStringStyle; +import static java.lang.System.err; +import static java.lang.System.out; +import static java.nio.channels.FileChannel.MapMode.READ_ONLY; +import static org.apache.commons.lang.builder.ToStringStyle.MULTI_LINE_STYLE; import java.io.ByteArrayInputStream; import java.io.File; @@ -32,6 +33,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; @@ -44,8 +46,7 @@ import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; -import static java.lang.System.err; -import static java.lang.System.out; +import org.apache.commons.lang.builder.ToStringBuilder; /** * This is a virtually unchanged DVN v2-3 implementation by @@ -134,7 +135,7 @@ public IngestableDataChecker() { /** * test this byte buffer against SPSS-SAV spec */ - public String testSAVformat(MappedByteBuffer buff) { + public String testSAVformat(ByteBuffer buff) { String result = null; buff.rewind(); boolean DEBUG = false; @@ -176,7 +177,7 @@ public String testSAVformat(MappedByteBuffer buff) { /** * test this byte buffer against STATA DTA spec */ - public String testDTAformat(MappedByteBuffer buff) { + public String testDTAformat(ByteBuffer buff) { String result = null; buff.rewind(); boolean DEBUG = false; @@ -294,7 +295,7 @@ public String testDTAformat(MappedByteBuffer buff) { /** * test this byte buffer against SAS Transport(XPT) spec */ - public String testXPTformat(MappedByteBuffer buff) { + public String testXPTformat(ByteBuffer buff) { String result = null; buff.rewind(); boolean DEBUG = false; @@ -341,7 +342,7 @@ public String testXPTformat(MappedByteBuffer buff) { /** * test this byte buffer against SPSS Portable (POR) spec */ - public String testPORformat(MappedByteBuffer buff) { + public String testPORformat(ByteBuffer buff) { String result = null; buff.rewind(); boolean DEBUG = false; @@ -506,7 +507,7 @@ public String testPORformat(MappedByteBuffer buff) { /** * test this byte buffer against R data file */ - public String testRDAformat(MappedByteBuffer buff) { + public String testRDAformat(ByteBuffer buff) { String result = null; buff.rewind(); @@ -590,18 +591,18 @@ public String testRDAformat(MappedByteBuffer buff) { public String detectTabularDataFormat(File fh) { boolean DEBUG = false; String readableFormatType = null; - FileChannel srcChannel = null; - FileInputStream inp = null; - try { + + + try (final FileInputStream inp = new FileInputStream(fh)) { + int buffer_size = this.getBufferSize(fh); dbgLog.fine("buffer_size: " + buffer_size); // set-up a FileChannel instance for a given file object - inp = new FileInputStream(fh); - srcChannel = inp.getChannel(); + final FileChannel srcChannel = inp.getChannel(); // create a read-only MappedByteBuffer - MappedByteBuffer buff = srcChannel.map(FileChannel.MapMode.READ_ONLY, 0, buffer_size); + MappedByteBuffer buff = srcChannel.map(READ_ONLY, 0, buffer_size); //this.printHexDump(buff, "hex dump of the byte-buffer"); @@ -661,10 +662,7 @@ public String detectTabularDataFormat(File fh) { } catch (IOException ie) { dbgLog.fine("other io exception detected"); ie.printStackTrace(); - } finally { - IOUtils.closeQuietly(srcChannel); - IOUtils.closeQuietly(inp); - } + } return readableFormatType; } @@ -709,7 +707,7 @@ private int getBufferSize(File fh) { return BUFFER_SIZE; } - private int getGzipBufferSize(MappedByteBuffer buff) { + private int getGzipBufferSize(ByteBuffer buff) { int GZIP_BUFFER_SIZE = 120; /* note: @@ -729,7 +727,7 @@ private int getGzipBufferSize(MappedByteBuffer buff) { /** * dump the data buffer in HEX */ - public void printHexDump(MappedByteBuffer buff, String hdr) { + public void printHexDump(ByteBuffer buff, String hdr) { int counter = 0; if (hdr != null) { out.println(hdr); @@ -751,7 +749,6 @@ public void printHexDump(MappedByteBuffer buff, String hdr) { @Override public String toString() { - return ToStringBuilder.reflectionToString(this, - ToStringStyle.MULTI_LINE_STYLE); + return ToStringBuilder.reflectionToString(this, MULTI_LINE_STYLE); } } diff --git a/dataverse-webapp/src/test/java/edu/harvard/iq/dataverse/ingest/IngestableDataCheckerTest.java b/dataverse-webapp/src/test/java/edu/harvard/iq/dataverse/ingest/IngestableDataCheckerTest.java index d7a0f368f0..4487de2fde 100644 --- a/dataverse-webapp/src/test/java/edu/harvard/iq/dataverse/ingest/IngestableDataCheckerTest.java +++ b/dataverse-webapp/src/test/java/edu/harvard/iq/dataverse/ingest/IngestableDataCheckerTest.java @@ -1,170 +1,61 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.ingest; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; +import static edu.harvard.iq.dataverse.ingest.IngestableDataChecker.STATA_13_HEADER; +import static org.assertj.core.api.Assertions.assertThat; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Path; +import java.nio.ByteBuffer; -import static org.junit.jupiter.api.Assertions.assertEquals; +import org.assertj.core.api.AbstractStringAssert; +import org.junit.jupiter.api.Test; -/** - * @author rmp553 - */ public class IngestableDataCheckerTest { - @TempDir - Path tempFolder; - - public IngestableDataCheckerTest() { - } - - @BeforeAll - public static void setUpClass() { - } - - @AfterAll - public static void tearDownClass() { - - } - - @BeforeEach - public void setUp() { - } - - @AfterEach - public void tearDown() { + private final IngestableDataChecker instance = new IngestableDataChecker(); + private ByteBuffer createBufferContaining(final String fileContents) + throws IOException { + final ByteBuffer buffer = ByteBuffer.allocate(100); + buffer.put(fileContents.getBytes()); + buffer.rewind(); + return buffer; } + private AbstractStringAssert assertThatDTAFormat(final String content) + throws Exception { + ByteBuffer buff = createBufferContaining(content); - private File createTempFile(String filename, String fileContents) throws IOException { - - if (filename == null) { - return null; - } - File fh = tempFolder.resolve(filename).toFile(); - fh.createNewFile(); - - if (fileContents != null) { - FileUtils.writeStringToFile(fh, fileContents); - } - - return fh; + return assertThat(this.instance.testDTAformat(buff)); } - private MappedByteBuffer createTempFileAndGetBuffer(String filename, String fileContents) throws IOException { + private AbstractStringAssert assertThatSAVFormat(final String content) + throws Exception { + ByteBuffer buff = createBufferContaining(content); - File fh = this.createTempFile(filename, fileContents); - - FileChannel srcChannel = new FileInputStream(fh).getChannel(); - - // create a read-only MappedByteBuffer - MappedByteBuffer buff = srcChannel.map(FileChannel.MapMode.READ_ONLY, 0, fh.length()); - - return buff; + return assertThat(this.instance.testSAVformat(buff)); } - private void msg(String m) { - System.out.println(m); + @Test + public void testADATAformat_returnsMimeType_forProperContent() throws Exception { + assertThatDTAFormat("l ").isEqualTo("application/x-stata"); + assertThatDTAFormat(STATA_13_HEADER).isEqualTo("application/x-stata-13"); } - - private void msgt(String m) { - msg("---------------------------"); - msg(m); - msg("---------------------------"); + @Test + public void testADATAformat_returnsNull_forBrokenContent() throws Exception { + assertThatDTAFormat("").isNull(); + assertThatDTAFormat("hello-non-stata-file-how-are-you").isNull(); } - /** - * Test of testDTAformat method, of class IngestableDataChecker. - * - * @throws java.io.IOException - */ @Test - public void testTestDTAformat(@TempDir Path tempFolder) throws IOException { - msgt("(1) testDTAformat"); - - msgt("(1a) Mock a Legit Stata File (application/x-stata)"); - MappedByteBuffer buff = createTempFileAndGetBuffer("testDTA.txt", "l "); - - IngestableDataChecker instance = new IngestableDataChecker(); - String result = instance.testDTAformat(buff); - msg("result 1a: " + result); - assertEquals(result, "application/x-stata"); - - - msgt("(1b) File is empty string (non-DTA)"); - buff = createTempFileAndGetBuffer("notDTA.txt", ""); - instance = new IngestableDataChecker(); - result = instance.testDTAformat(buff); - msg("result 1b: " + result); - assertEquals(result, null); - - - msgt("(1c) File is some random text (non-DTA)"); - buff = createTempFileAndGetBuffer("notDTA2.txt", "hello-non-stata-file-how-are-you"); - instance = new IngestableDataChecker(); - result = instance.testDTAformat(buff); - msg("result 1c: " + result); - assertEquals(result, null); - - - msgt("(1d) Mock a Legit Stata File with STATA_13_HEADER"); - buff = createTempFileAndGetBuffer("testDTA2.txt", IngestableDataChecker.STATA_13_HEADER); - result = instance.testDTAformat(buff); - msg("result 1d: " + result); - assertEquals(result, "application/x-stata-13"); - - + public void testSAVformat_returnsMimeType_forProperContent() throws Exception { + assertThatSAVFormat("$FL2").isEqualTo("application/x-spss-sav"); } - - /** - * Test of testSAVformat method, of class IngestableDataChecker. - */ @Test - public void testTestSAVformat() throws IOException { - msgt("(2) testSAVformat"); - - msgt("(2a) Mock a Legit SPSS-SAV File (application/x-spss-sav)"); - MappedByteBuffer buff = createTempFileAndGetBuffer("testSAV.txt", "$FL2"); - - IngestableDataChecker instance = new IngestableDataChecker(); - String result = instance.testSAVformat(buff); - msg("result 2a: " + result); - assertEquals(result, "application/x-spss-sav"); - - msgt("(2b) File is empty string"); - buff = createTempFileAndGetBuffer("testNotSAV-empty.txt", ""); - - instance = new IngestableDataChecker(); - result = instance.testSAVformat(buff); - msg("result 2b: " + result); - assertEquals(result, null); - - msgt("(2c) File is non-SAV string"); - buff = createTempFileAndGetBuffer("testNotSAV-string.txt", "i-am-not-a-x-spss-sav-file"); - instance = new IngestableDataChecker(); - result = instance.testSAVformat(buff); - msg("result 2c: " + result); - assertEquals(result, null); - + public void testSAVformat_returnsNull_forBrokenContent() throws Exception { + assertThatSAVFormat("").isNull(); + assertThatSAVFormat("i-am-not-a-x-spss-sav-file").isNull(); } - }