Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into #1953_APKParser
Browse files Browse the repository at this point in the history
  • Loading branch information
@wladimirleite
wladimirleite committed Nov 1, 2023
2 parents 8911e11 + df5950c commit 8476ce9
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 35 deletions.
110 changes: 86 additions & 24 deletions iped-engine/src/main/java/iped/engine/data/Item.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package iped.engine.data;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
Expand All @@ -21,6 +21,7 @@
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.commons.compress.utils.SeekableInMemoryByteChannel;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
Expand Down Expand Up @@ -194,6 +195,8 @@ public static void setStartID(int start) {

private byte[] thumb;

private byte[] data;

private ISeekableInputStreamFactory inputStreamFactory;

static final int BUF_LEN = 8 * 1024 * 1024;
Expand Down Expand Up @@ -244,6 +247,7 @@ public void dispose(boolean clearTextCache) {
} catch (Exception e) {
LOGGER.warn("Error closing resources of " + getPath(), e);
}
data = null;
tmpFile = null;
tis = null;
try {
Expand All @@ -263,12 +267,7 @@ public Date getAccessDate() {
return accessDate;
}

/**
* @return um BufferedInputStream com o conteúdo do item
* @throws IOException
*/
public BufferedInputStream getBufferedInputStream() throws IOException {

private int getBestBufferSize() {
int len = 8192;
if (length != null && length > len) {
if (length < BUF_LEN) {
Expand All @@ -277,8 +276,18 @@ public BufferedInputStream getBufferedInputStream() throws IOException {
len = BUF_LEN;
}
}
return len;
}

return new BufferedInputStream(getSeekableInputStream(), len);
/**
* @return um BufferedInputStream com o conteúdo do item
* @throws IOException
*/
public BufferedInputStream getBufferedInputStream() throws IOException {
if (data != null) {
return new BufferedInputStream(new ByteArrayInputStream(data));
}
return new BufferedInputStream(getSeekableInputStream(), getBestBufferSize());
}

/**
Expand Down Expand Up @@ -527,6 +536,10 @@ public String getPath() {
@Override
public SeekableInputStream getSeekableInputStream() throws IOException {

if (data != null) {
return new SeekableFileInputStream(new SeekableInMemoryByteChannel(data));
}

// block 1 (referenciado abaixo)
if (tmpFile == null && tis != null && tis.hasFile()) {
tmpFile = tis.getFile();
Expand Down Expand Up @@ -574,9 +587,41 @@ public SeekableInputStream getSeekableInputStream() throws IOException {

@Override
public SeekableByteChannel getSeekableByteChannel() throws IOException {
if (data != null) {
return new SeekableInMemoryByteChannel(data);
}
return new SeekableByteChannelImpl(this.getSeekableInputStream());
}

/**
* Cache data in memory for small items to avoid:
* 1. multiple decompression of data from compressed evidences
* 2. multiple reads from evidences in network shares
* 3. writing small temp files in temp dir, when possible
* 4. decrease heavy IO calls into kernel space
*
* @return true if data was cached on memory, false otherwise
*/
public boolean cacheDataInMemory() {
if (length == null || length > BUF_LEN) {
return false;
}
if (data != null) {
return true;
}
try (InputStream is = this.getSeekableInputStream()) {
data = new byte[length.intValue()];
int i, offset = 0;
while (offset < data.length && (i = is.read(data, offset, data.length - offset)) != -1) {
offset += i;
}
return true;
} catch (IOException e) {
// ignore
}
return false;
}

/**
* Usado em módulos que só possam processar um File e não um InputStream. Pode
* impactar performance pois gera arquivo temporário.
Expand All @@ -593,23 +638,34 @@ public File getTempFile() throws IOException {
if (tis != null && tis.hasFile()) {
tmpFile = tis.getFile();
} else {
String ext = ".tmp"; //$NON-NLS-1$
if (type != null && !type.toString().isEmpty()) {
ext = Util.getValidFilename("." + type.toString()); //$NON-NLS-1$
}
final Path path = Files.createTempFile("iped", ext); //$NON-NLS-1$
tmpResources.addResource(new Closeable() {
public void close() throws IOException {
Files.delete(path);
try (SeekableInputStream sis = getSeekableInputStream()) {
if (sis instanceof SeekableFileInputStream) {
File file = ((SeekableFileInputStream) sis).getFile();
if (file != null && IOUtil.isTemporaryFile(file)) {
tmpFile = file;
}
}
});

try (InputStream in = getBufferedInputStream()) {
Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING);
if (tmpFile == null) {
String ext = ".tmp"; //$NON-NLS-1$
if (type != null && !type.toString().isEmpty()) {
ext = Util.getValidFilename("." + type.toString()); //$NON-NLS-1$
}
Path path = Files.createTempFile("iped", ext); //$NON-NLS-1$
if (data != null) {
Files.write(path, data);
} else {
Files.copy(new BufferedInputStream(sis, getBestBufferSize()), path, StandardCopyOption.REPLACE_EXISTING);
}
tmpFile = path.toFile();
}
final File finalFile = tmpFile;
tmpResources.addResource(new Closeable() {
public void close() {
finalFile.delete();
}
});
}
tmpFile = path.toFile();
}

}
return tmpFile;
}
Expand All @@ -636,14 +692,20 @@ public TikaInputStream getTikaStream() throws IOException {
if (tmpFile == null && tis != null && tis.hasFile()) {
tmpFile = tis.getFile();
}
// reset tis, it may have been set (and consumed) by a previous call of this
// method
tis = null;
if (tmpFile != null) {
try {
tis = TikaInputStream.get(tmpFile.toPath());
} catch (FileNotFoundException fnfe) {
} catch (IOException fnfe) {
tmpFile = null;
}
}
if (tmpFile == null) {
if (tis == null && data != null) {
tis = TikaInputStream.get(data);
}
if (tis == null) {
tis = TikaInputStream.get(getBufferedInputStream());
}
}
Expand Down
12 changes: 9 additions & 3 deletions iped-engine/src/main/java/iped/engine/io/ZIPInputStreamFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.nio.channels.ClosedChannelException;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
Expand Down Expand Up @@ -159,7 +160,12 @@ public SeekableInputStream getSeekableInputStream(String path) throws IOExceptio
synchronized (filesCache) {
tmp = filesCache.get(path);
if (tmp != null) {
return new SeekableFileInputStream(tmp.toFile());
try {
return new SeekableFileInputStream(tmp.toFile());
} catch (NoSuchFileException e) {
// Could have been deleted by Item.dispose()
filesCache.remove(path);
}
}
}

Expand Down Expand Up @@ -194,7 +200,7 @@ public Pair<Path, byte[]> call() throws Exception {
if (zae.getSize() <= MAX_BYTES_CACHED) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
int read;
byte[] buf = new byte[8192];
byte[] buf = new byte[UFDR_BUF_SIZE];
while ((read = is.read(buf, 0, buf.length)) >= 0) {
if (canceled.get()) {
return null;
Expand All @@ -212,7 +218,7 @@ public Pair<Path, byte[]> call() throws Exception {
tmp = Files.createTempFile("zip-stream", null);
try (OutputStream out = Files.newOutputStream(tmp)) {
int read;
byte[] buf = new byte[8192];
byte[] buf = new byte[UFDR_BUF_SIZE];
while (!canceled.get() && (read = is.read(buf, 0, buf.length)) >= 0) {
out.write(buf, 0, read);
}
Expand Down
30 changes: 22 additions & 8 deletions iped-engine/src/main/java/iped/engine/task/TempFileTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import iped.configuration.Configurable;
import iped.data.IItem;
import iped.engine.CmdLineArgs;
import iped.engine.config.ConfigurationManager;
import iped.engine.config.LocalConfig;
import iped.engine.data.Item;
Expand All @@ -30,10 +31,11 @@ public class TempFileTask extends AbstractTask {
private static Logger LOGGER = LoggerFactory.getLogger(TempFileTask.class);
private static int MAX_TEMPFILE_LEN = 1024 * 1024 * 1024;
private boolean indexTempOnSSD = false;
private boolean isEnabled = true;

@Override
public boolean isEnabled() {
return indexTempOnSSD;
return isEnabled;
}

@Override
Expand All @@ -45,7 +47,8 @@ public List<Configurable<?>> getConfigurables() {
public void init(ConfigurationManager configurationManager) throws Exception {
LocalConfig config = configurationManager.findObject(LocalConfig.class);
indexTempOnSSD = config.isIndexTempOnSSD();

CmdLineArgs args = (CmdLineArgs) caseData.getCaseObject(CmdLineArgs.class.getName());
isEnabled = !"fastmode".equals(args.getProfile()) && !"triage".equals(args.getProfile());
}

@Override
Expand All @@ -56,14 +59,25 @@ public void finish() throws Exception {
@Override
protected void process(IItem evidence) throws Exception {


if (evidence instanceof Item) {
if (((Item) evidence).cacheDataInMemory()) {
return;
}
}

if (!indexTempOnSSD) {
return;
}

Long len = evidence.getLength();
if (indexTempOnSSD && len != null
&& len <= MAX_TEMPFILE_LEN /* && evidence.getPath().toLowerCase().contains(".e01/vol_vol") */) {
if (len != null && len <= MAX_TEMPFILE_LEN) {
try {
if (!IOUtil.hasFile(evidence) && !evidence.isSubItem()
// skip carved items pointing to parent temp file
&& (!(evidence instanceof Item) || !((Item) evidence).hasParentTmpFile())) {
evidence.getTempFile();
// skip items with File refs && carved items pointing to parent temp file
if (!IOUtil.hasFile(evidence) && (!(evidence instanceof Item) || !((Item) evidence).hasParentTmpFile())) {
if (!evidence.isSubItem()) { // should we create temp files for subitems compressed into the sqlite storages?
evidence.getTempFile();
}
}
} catch (IOException e) {
LOGGER.warn("{} Error creating temp file {} {}", Thread.currentThread().getName(), evidence.getPath(), //$NON-NLS-1$
Expand Down
6 changes: 6 additions & 0 deletions iped-utils/src/main/java/iped/utils/SeekableFileInputStream.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,23 @@

public class SeekableFileInputStream extends SeekableInputStream {

private File file;
private SeekableByteChannel sbc;
private boolean closed = false;

public SeekableFileInputStream(File file) throws IOException {
this.file = file;
this.sbc = FileChannel.open(file.toPath(), StandardOpenOption.READ);
}

public SeekableFileInputStream(SeekableByteChannel channel) {
this.sbc = channel;
}

public File getFile() {
return this.file;
}

@Override
public int read(byte b[]) throws IOException {
return read(b, 0, b.length);
Expand Down

0 comments on commit 8476ce9

Please sign in to comment.