diff --git a/iped-engine/src/main/java/iped/engine/data/Item.java b/iped-engine/src/main/java/iped/engine/data/Item.java index 35e9e721ca..7213c518c4 100644 --- a/iped-engine/src/main/java/iped/engine/data/Item.java +++ b/iped-engine/src/main/java/iped/engine/data/Item.java @@ -1,9 +1,9 @@ package iped.engine.data; import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.Reader; @@ -21,6 +21,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.compress.utils.SeekableInMemoryByteChannel; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -194,6 +195,8 @@ public static void setStartID(int start) { private byte[] thumb; + private byte[] data; + private ISeekableInputStreamFactory inputStreamFactory; static final int BUF_LEN = 8 * 1024 * 1024; @@ -244,6 +247,7 @@ public void dispose(boolean clearTextCache) { } catch (Exception e) { LOGGER.warn("Error closing resources of " + getPath(), e); } + data = null; tmpFile = null; tis = null; try { @@ -263,12 +267,7 @@ public Date getAccessDate() { return accessDate; } - /** - * @return um BufferedInputStream com o conteúdo do item - * @throws IOException - */ - public BufferedInputStream getBufferedInputStream() throws IOException { - + private int getBestBufferSize() { int len = 8192; if (length != null && length > len) { if (length < BUF_LEN) { @@ -277,8 +276,18 @@ public BufferedInputStream getBufferedInputStream() throws IOException { len = BUF_LEN; } } + return len; + } - return new BufferedInputStream(getSeekableInputStream(), len); + /** + * @return um BufferedInputStream com o conteúdo do item + * @throws IOException + */ + public BufferedInputStream getBufferedInputStream() throws IOException { + if (data != null) { + return new BufferedInputStream(new ByteArrayInputStream(data)); + } + return new BufferedInputStream(getSeekableInputStream(), getBestBufferSize()); } /** @@ -527,6 +536,10 @@ public String getPath() { @Override public SeekableInputStream getSeekableInputStream() throws IOException { + if (data != null) { + return new SeekableFileInputStream(new SeekableInMemoryByteChannel(data)); + } + // block 1 (referenciado abaixo) if (tmpFile == null && tis != null && tis.hasFile()) { tmpFile = tis.getFile(); @@ -574,9 +587,41 @@ public SeekableInputStream getSeekableInputStream() throws IOException { @Override public SeekableByteChannel getSeekableByteChannel() throws IOException { + if (data != null) { + return new SeekableInMemoryByteChannel(data); + } return new SeekableByteChannelImpl(this.getSeekableInputStream()); } + /** + * Cache data in memory for small items to avoid: + * 1. multiple decompression of data from compressed evidences + * 2. multiple reads from evidences in network shares + * 3. writing small temp files in temp dir, when possible + * 4. decrease heavy IO calls into kernel space + * + * @return true if data was cached on memory, false otherwise + */ + public boolean cacheDataInMemory() { + if (length == null || length > BUF_LEN) { + return false; + } + if (data != null) { + return true; + } + try (InputStream is = this.getSeekableInputStream()) { + data = new byte[length.intValue()]; + int i, offset = 0; + while (offset < data.length && (i = is.read(data, offset, data.length - offset)) != -1) { + offset += i; + } + return true; + } catch (IOException e) { + // ignore + } + return false; + } + /** * Usado em módulos que só possam processar um File e não um InputStream. Pode * impactar performance pois gera arquivo temporário. @@ -593,23 +638,34 @@ public File getTempFile() throws IOException { if (tis != null && tis.hasFile()) { tmpFile = tis.getFile(); } else { - String ext = ".tmp"; //$NON-NLS-1$ - if (type != null && !type.toString().isEmpty()) { - ext = Util.getValidFilename("." + type.toString()); //$NON-NLS-1$ - } - final Path path = Files.createTempFile("iped", ext); //$NON-NLS-1$ - tmpResources.addResource(new Closeable() { - public void close() throws IOException { - Files.delete(path); + try (SeekableInputStream sis = getSeekableInputStream()) { + if (sis instanceof SeekableFileInputStream) { + File file = ((SeekableFileInputStream) sis).getFile(); + if (file != null && IOUtil.isTemporaryFile(file)) { + tmpFile = file; + } } - }); - - try (InputStream in = getBufferedInputStream()) { - Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); + if (tmpFile == null) { + String ext = ".tmp"; //$NON-NLS-1$ + if (type != null && !type.toString().isEmpty()) { + ext = Util.getValidFilename("." + type.toString()); //$NON-NLS-1$ + } + Path path = Files.createTempFile("iped", ext); //$NON-NLS-1$ + if (data != null) { + Files.write(path, data); + } else { + Files.copy(new BufferedInputStream(sis, getBestBufferSize()), path, StandardCopyOption.REPLACE_EXISTING); + } + tmpFile = path.toFile(); + } + final File finalFile = tmpFile; + tmpResources.addResource(new Closeable() { + public void close() { + finalFile.delete(); + } + }); } - tmpFile = path.toFile(); } - } return tmpFile; } @@ -636,14 +692,20 @@ public TikaInputStream getTikaStream() throws IOException { if (tmpFile == null && tis != null && tis.hasFile()) { tmpFile = tis.getFile(); } + // reset tis, it may have been set (and consumed) by a previous call of this + // method + tis = null; if (tmpFile != null) { try { tis = TikaInputStream.get(tmpFile.toPath()); - } catch (FileNotFoundException fnfe) { + } catch (IOException fnfe) { tmpFile = null; } } - if (tmpFile == null) { + if (tis == null && data != null) { + tis = TikaInputStream.get(data); + } + if (tis == null) { tis = TikaInputStream.get(getBufferedInputStream()); } } diff --git a/iped-engine/src/main/java/iped/engine/io/ZIPInputStreamFactory.java b/iped-engine/src/main/java/iped/engine/io/ZIPInputStreamFactory.java index 27b33eb156..0447f0100d 100644 --- a/iped-engine/src/main/java/iped/engine/io/ZIPInputStreamFactory.java +++ b/iped-engine/src/main/java/iped/engine/io/ZIPInputStreamFactory.java @@ -9,6 +9,7 @@ import java.nio.channels.ClosedChannelException; import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -159,7 +160,12 @@ public SeekableInputStream getSeekableInputStream(String path) throws IOExceptio synchronized (filesCache) { tmp = filesCache.get(path); if (tmp != null) { - return new SeekableFileInputStream(tmp.toFile()); + try { + return new SeekableFileInputStream(tmp.toFile()); + } catch (NoSuchFileException e) { + // Could have been deleted by Item.dispose() + filesCache.remove(path); + } } } @@ -194,7 +200,7 @@ public Pair call() throws Exception { if (zae.getSize() <= MAX_BYTES_CACHED) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); int read; - byte[] buf = new byte[8192]; + byte[] buf = new byte[UFDR_BUF_SIZE]; while ((read = is.read(buf, 0, buf.length)) >= 0) { if (canceled.get()) { return null; @@ -212,7 +218,7 @@ public Pair call() throws Exception { tmp = Files.createTempFile("zip-stream", null); try (OutputStream out = Files.newOutputStream(tmp)) { int read; - byte[] buf = new byte[8192]; + byte[] buf = new byte[UFDR_BUF_SIZE]; while (!canceled.get() && (read = is.read(buf, 0, buf.length)) >= 0) { out.write(buf, 0, read); } diff --git a/iped-engine/src/main/java/iped/engine/task/TempFileTask.java b/iped-engine/src/main/java/iped/engine/task/TempFileTask.java index 7c0c87c12b..508e286902 100644 --- a/iped-engine/src/main/java/iped/engine/task/TempFileTask.java +++ b/iped-engine/src/main/java/iped/engine/task/TempFileTask.java @@ -9,6 +9,7 @@ import iped.configuration.Configurable; import iped.data.IItem; +import iped.engine.CmdLineArgs; import iped.engine.config.ConfigurationManager; import iped.engine.config.LocalConfig; import iped.engine.data.Item; @@ -30,10 +31,11 @@ public class TempFileTask extends AbstractTask { private static Logger LOGGER = LoggerFactory.getLogger(TempFileTask.class); private static int MAX_TEMPFILE_LEN = 1024 * 1024 * 1024; private boolean indexTempOnSSD = false; + private boolean isEnabled = true; @Override public boolean isEnabled() { - return indexTempOnSSD; + return isEnabled; } @Override @@ -45,7 +47,8 @@ public List> getConfigurables() { public void init(ConfigurationManager configurationManager) throws Exception { LocalConfig config = configurationManager.findObject(LocalConfig.class); indexTempOnSSD = config.isIndexTempOnSSD(); - + CmdLineArgs args = (CmdLineArgs) caseData.getCaseObject(CmdLineArgs.class.getName()); + isEnabled = !"fastmode".equals(args.getProfile()) && !"triage".equals(args.getProfile()); } @Override @@ -56,14 +59,25 @@ public void finish() throws Exception { @Override protected void process(IItem evidence) throws Exception { + + if (evidence instanceof Item) { + if (((Item) evidence).cacheDataInMemory()) { + return; + } + } + + if (!indexTempOnSSD) { + return; + } + Long len = evidence.getLength(); - if (indexTempOnSSD && len != null - && len <= MAX_TEMPFILE_LEN /* && evidence.getPath().toLowerCase().contains(".e01/vol_vol") */) { + if (len != null && len <= MAX_TEMPFILE_LEN) { try { - if (!IOUtil.hasFile(evidence) && !evidence.isSubItem() - // skip carved items pointing to parent temp file - && (!(evidence instanceof Item) || !((Item) evidence).hasParentTmpFile())) { - evidence.getTempFile(); + // skip items with File refs && carved items pointing to parent temp file + if (!IOUtil.hasFile(evidence) && (!(evidence instanceof Item) || !((Item) evidence).hasParentTmpFile())) { + if (!evidence.isSubItem()) { // should we create temp files for subitems compressed into the sqlite storages? + evidence.getTempFile(); + } } } catch (IOException e) { LOGGER.warn("{} Error creating temp file {} {}", Thread.currentThread().getName(), evidence.getPath(), //$NON-NLS-1$ diff --git a/iped-utils/src/main/java/iped/utils/SeekableFileInputStream.java b/iped-utils/src/main/java/iped/utils/SeekableFileInputStream.java index 8a00ec1415..2d47fda576 100644 --- a/iped-utils/src/main/java/iped/utils/SeekableFileInputStream.java +++ b/iped-utils/src/main/java/iped/utils/SeekableFileInputStream.java @@ -11,10 +11,12 @@ public class SeekableFileInputStream extends SeekableInputStream { + private File file; private SeekableByteChannel sbc; private boolean closed = false; public SeekableFileInputStream(File file) throws IOException { + this.file = file; this.sbc = FileChannel.open(file.toPath(), StandardOpenOption.READ); } @@ -22,6 +24,10 @@ public SeekableFileInputStream(SeekableByteChannel channel) { this.sbc = channel; } + public File getFile() { + return this.file; + } + @Override public int read(byte b[]) throws IOException { return read(b, 0, b.length);