diff --git a/iped-app/src/main/java/iped/app/ui/ExternalFileOpen.java b/iped-app/src/main/java/iped/app/ui/ExternalFileOpen.java index 417a3aa11f..f7cc6f0375 100644 --- a/iped-app/src/main/java/iped/app/ui/ExternalFileOpen.java +++ b/iped-app/src/main/java/iped/app/ui/ExternalFileOpen.java @@ -35,11 +35,7 @@ public void run() { try { if (IOUtil.isToOpenExternally(item.getName(), item.getType())) { LOGGER.info("Externally Opening file " + item.getPath()); //$NON-NLS-1$ - File file = Util.getFileRenamedToExt(item.getTempFile(), item.getType()); - file.setReadOnly(); - if (IOUtil.isTemporaryFile(file)) { - file.deleteOnExit(); - } + File file = Util.getFileWithRightExt(item); open(file); } diff --git a/iped-engine/src/main/java/iped/engine/data/IPEDSource.java b/iped-engine/src/main/java/iped/engine/data/IPEDSource.java index 5dc82e1305..d7027c3be3 100644 --- a/iped-engine/src/main/java/iped/engine/data/IPEDSource.java +++ b/iped-engine/src/main/java/iped/engine/data/IPEDSource.java @@ -132,6 +132,8 @@ public class IPEDSource implements IIPEDSource { boolean isReport = false; + boolean askImagePathIfNotFound = true; + public static boolean checkIfIsCaseFolder(File dir) { File module = new File(dir, MODULE_DIR); if (new File(module, INDEX_DIR).exists() && new File(module, LIB_DIR).exists() && new File(module, DATA_DIR).exists()) { @@ -155,7 +157,11 @@ public IPEDSource(File casePath) { } public IPEDSource(File casePath, IndexWriter iw) { + this(casePath, iw, true); + } + public IPEDSource(File casePath, IndexWriter iw, boolean askImagePathIfNotFound) { + this.askImagePathIfNotFound = askImagePathIfNotFound; this.casePath = casePath; moduleDir = new File(casePath, MODULE_DIR); index = new File(moduleDir, INDEX_DIR); @@ -239,6 +245,9 @@ public IPEDSource(File casePath, IndexWriter iw) { multiBookmarks = new MultiBookmarks(Collections.singletonList(this)); } catch (Exception e) { + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } throw new RuntimeException(e.getMessage(), e); } } @@ -601,8 +610,13 @@ private void updateImagePathsToAbsolute(File casePath, File sleuthFile) throws E if (newPaths.size() > 0) { testCanWriteToCase(sleuthFile); sleuthCase.setImagePaths(id, newPaths); - } else if (iw == null) - askNewImagePath(id, paths, sleuthFile); + } else if (iw == null) { + if (askImagePathIfNotFound) { + askNewImagePath(id, paths, sleuthFile); + } else { + throw new RuntimeException("Image not found: " + paths.get(0)); + } + } } } diff --git a/iped-engine/src/main/java/iped/engine/lucene/ConfiguredFSDirectory.java b/iped-engine/src/main/java/iped/engine/lucene/ConfiguredFSDirectory.java index 6af9a64889..14c86d03f9 100644 --- a/iped-engine/src/main/java/iped/engine/lucene/ConfiguredFSDirectory.java +++ b/iped-engine/src/main/java/iped/engine/lucene/ConfiguredFSDirectory.java @@ -19,7 +19,7 @@ public static FSDirectory open(File indexDir) throws IOException { IndexTaskConfig config = ConfigurationManager.get().findObject(IndexTaskConfig.class); FSDirectory result; - if (config.isUseNIOFSDirectory()) { + if (config != null && config.isUseNIOFSDirectory()) { result = new NIOFSDirectory(indexDir.toPath()); } else { result = FSDirectory.open(indexDir.toPath()); diff --git a/iped-engine/src/main/java/iped/engine/task/ParsingTask.java b/iped-engine/src/main/java/iped/engine/task/ParsingTask.java index b912c06d47..e149f5d0b2 100644 --- a/iped-engine/src/main/java/iped/engine/task/ParsingTask.java +++ b/iped-engine/src/main/java/iped/engine/task/ParsingTask.java @@ -29,10 +29,12 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.tika.exception.TikaException; import org.apache.tika.extractor.EmbeddedDocumentExtractor; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; @@ -45,6 +47,7 @@ import org.apache.tika.parser.Parser; import org.apache.tika.parser.html.HtmlMapper; import org.apache.tika.parser.html.IdentityHtmlMapper; +import org.apache.tika.utils.XMLReaderUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.xml.sax.ContentHandler; @@ -159,6 +162,7 @@ public class ParsingTask extends ThumbTask implements EmbeddedDocumentExtractor private static final Set typesToCheckZipBomb = getTypesToCheckZipbomb(); private static AtomicInteger containersBeingExpanded = new AtomicInteger(); + private static AtomicBoolean tikaSAXPoolSizeSet = new AtomicBoolean(false); private CategoryToExpandConfig expandConfig; private ParsingTaskConfig parsingConfig; @@ -237,8 +241,6 @@ private ParseContext getTikaContext(File output, IPEDSource ipedsource) { context.set(ArchiveStreamFactory.class, new ArchiveStreamFactory("Cp850")); //$NON-NLS-1$ // Indexa conteudo de todos os elementos de HTMLs, como script, etc context.set(HtmlMapper.class, IdentityHtmlMapper.INSTANCE); - // we have seen very large records in valid docs - org.apache.poi.hpsf.CodePageString.setMaxRecordLength(512_000); context.set(IStreamSource.class, evidence); context.set(IItemReader.class, evidence); @@ -791,6 +793,18 @@ public static void setupParsingOptions(ConfigurationManager configurationManager ParsersConfig parserConfig = configurationManager.findObject(ParsersConfig.class); System.setProperty("tika.config", parserConfig.getTmpConfigFile().getAbsolutePath()); + // we have seen very large records in valid docs + org.apache.poi.hpsf.CodePageString.setMaxRecordLength(512_000); + + // heavy Tika configuration + if (!tikaSAXPoolSizeSet.getAndSet(true)) { + try { + XMLReaderUtils.setPoolSize(Runtime.getRuntime().availableProcessors()); + } catch (TikaException e) { + e.printStackTrace(); + } + } + // most options below are set using sys props because they are also used by // child external processes diff --git a/iped-engine/src/main/java/iped/engine/util/IPEDCrawler.java b/iped-engine/src/main/java/iped/engine/util/IPEDCrawler.java new file mode 100644 index 0000000000..b4a45c91c7 --- /dev/null +++ b/iped-engine/src/main/java/iped/engine/util/IPEDCrawler.java @@ -0,0 +1,161 @@ +package iped.engine.util; + +import java.io.File; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +import iped.data.IItem; +import iped.engine.data.IPEDSource; +import iped.engine.search.IPEDSearcher; + +public class IPEDCrawler { + + private static final boolean SKIP_KNOWN_FOLDERS = true; + + private static ConcurrentLinkedQueue cases = new ConcurrentLinkedQueue<>(); + private static AtomicInteger numCases = new AtomicInteger(); + + public static void main(String[] args) { + + if (args.length != 3) { + System.err.println("Please provide exactly 3 parameters: input_folder export_folder search_query"); + System.exit(1); + } + + File folderToScan = new File(args[0]); + File exportFolder = new File(args[1]); + String query = args[2]; + + if (!folderToScan.exists() || !folderToScan.isDirectory()) { + System.err.println("Input cases folder doesn't exist or is not a directory!"); + System.exit(2); + } + + exportFolder.mkdirs(); + + if (!exportFolder.exists() || !exportFolder.isDirectory()) { + System.err.println("Export folder couldn't be created or is not a directory!"); + System.exit(3); + } + + Thread folderScan = searchCasesinFolder(folderToScan); + + AtomicInteger counter = new AtomicInteger(); + AtomicInteger exported = new AtomicInteger(); + AtomicInteger finished = new AtomicInteger(); + ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + + boolean first = true; + while (folderScan.isAlive() || !cases.isEmpty()) { + File file = cases.poll(); + if (file == null) { + continue; + } + if (first) { + // initialize sleuthkit using just one thread + System.out.println("Initializing from case " + file.getAbsolutePath()); + IPEDSource ipedCase = new IPEDSource(file, null, false); + ipedCase.close(); + first = false; + } + executor.execute(new Runnable() { + @Override + public void run() { + int caseNum = counter.incrementAndGet(); + System.out.println("Searching for files into case " + caseNum + ": " + file.getAbsolutePath()); + try (IPEDSource ipedCase = new IPEDSource(file, null, false)) { + IPEDSearcher searcher = new IPEDSearcher(ipedCase, query); + int[] itemIds = searcher.search().getIds(); + System.out.println("Found " + itemIds.length + " files."); + if (itemIds.length == 0) { + return; + } + System.out.println("Exporting..."); + for (Integer id : itemIds) { + IItem item = ipedCase.getItemByID(id); + File parentDir = new File(exportFolder, "case_" + caseNum); + parentDir.mkdirs(); + File target = new File(parentDir, Util.getValidFilename(Util.getNameWithTrueExt(item))); + String ext = ""; + int idx, suffix = 0; + if ((idx = target.getName().lastIndexOf('.')) > -1) { + ext = target.getName().substring(idx); + } else { + idx = target.getName().length(); + } + synchronized (exported) { + while (target.exists()) { + target = new File(parentDir, target.getName().substring(0, idx) + (++suffix) + ext); + } + target.createNewFile(); + } + try (InputStream in = item.getBufferedInputStream()) { + Files.copy(in, target.toPath(), StandardCopyOption.REPLACE_EXISTING); + exported.getAndIncrement(); + } catch (Exception e0) { + e0.printStackTrace(); + } + } + System.out.println("Exported " + exported + " files."); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + finished.incrementAndGet(); + } + } + }); + + } + while (finished.get() < numCases.get()) { + continue; + } + System.out.println("Exported " + exported + " files."); + System.exit(0); + + } + + private static Thread searchCasesinFolder(File folder) { + Thread t = new Thread() { + public void run() { + recurse(folder); + System.out.println("Cases found: " + numCases.get()); + } + }; + t.start(); + return t; + } + + private static void recurse(File folder) { + if (SKIP_KNOWN_FOLDERS) { + String name = folder.getName(); + if (((name.equals("Exportados") || name.equals("Exported")) && (new File(folder.getParentFile(), IPEDSource.MODULE_DIR).exists() || new File(folder.getParentFile(), "indexador").exists())) || + ((name.equals("report") || name.equals("relatorio")) && new File(folder, "thumbs").exists()) || + (name.equals("indexador") && new File(folder, IPEDSource.INDEX_DIR).exists())) { + return; + } + } + if (new File(folder, IPEDSource.MODULE_DIR + "/" + IPEDSource.INDEX_DIR).exists()) { + System.out.println("Case found in " + folder.getAbsolutePath()); + cases.add(folder); + numCases.incrementAndGet(); + } else { + System.out.println("Searching for cases in " + folder.getAbsolutePath()); + File[] subFiles = folder.listFiles(); + if (subFiles != null) { + for (File file : subFiles) { + if (file.isDirectory()) { + recurse(file); + } + } + } + } + } + +} + diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/chrome/ChromeSqliteParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/chrome/ChromeSqliteParser.java index 6b7ab47273..789d17e9d8 100755 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/chrome/ChromeSqliteParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/chrome/ChromeSqliteParser.java @@ -9,7 +9,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -498,7 +498,7 @@ private void parseChromeSearches(ContentHandler handler, Metadata metadata, Pars protected List getResumedHistory(Connection connection, Metadata metadata, ParseContext context) throws SQLException { - List resumedHistory = new LinkedList(); + List resumedHistory = new ArrayList(); Statement st = null; try { @@ -524,7 +524,7 @@ protected List getResumedHistory(Connection connection, Metadata m protected List getHistory(Connection connection, Metadata metadata, ParseContext context) throws SQLException { - List history = new LinkedList(); + List history = new ArrayList(); Statement st = null; try { @@ -549,7 +549,7 @@ protected List getHistory(Connection connection, Metadata metadata, Parse protected List getDownloads(Connection connection, Metadata metadata, ParseContext context) throws SQLException { - List downloads = new LinkedList(); + List downloads = new ArrayList(); Statement st = null; try { @@ -582,7 +582,7 @@ protected List getDownloads(Connection connection, Metadata metadata, protected List getSearchTerms(Connection connection, Metadata metadata, ParseContext context) throws SQLException { - List searches = new LinkedList(); + List searches = new ArrayList(); Statement st = null; try { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/edge/EdgeWebCacheParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/edge/EdgeWebCacheParser.java index 1db90a3744..c68b31e91e 100755 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/edge/EdgeWebCacheParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/edge/EdgeWebCacheParser.java @@ -7,9 +7,9 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.StandardCopyOption; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -287,7 +287,7 @@ private void parseEdgeHistory(ContentHandler handler, Metadata metadata, ParseCo protected List getHistory(String filePath, PointerByReference filePointerReference, ItemInfo itemInfo) throws EdgeWebCacheException { - List history = new LinkedList(); + List history = new ArrayList(); try { /* diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/firefox/FirefoxSqliteParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/firefox/FirefoxSqliteParser.java index 017a1aea20..35b0ee37c6 100755 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/firefox/FirefoxSqliteParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/firefox/FirefoxSqliteParser.java @@ -9,7 +9,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -504,7 +504,7 @@ private void parseFirefoxResumedHistory(ContentHandler handler, Metadata metadat protected List getResumedHistory(Connection connection, Metadata metadata, ParseContext context) throws SQLException { - List resumedHistory = new LinkedList(); + List resumedHistory = new ArrayList(); Statement st = null; try { @@ -531,7 +531,7 @@ protected List getResumedHistory(Connection connection, Metadata m protected List getHistory(Connection connection, Metadata metadata, ParseContext context) throws SQLException { - List history = new LinkedList(); + List history = new ArrayList(); Statement st = null; try { @@ -555,7 +555,7 @@ protected List getHistory(Connection connection, Metadata metadata, Parse protected List getBookmarks(Connection connection, Metadata metadata, ParseContext context) throws SQLException { - List bookmarks = new LinkedList(); + List bookmarks = new ArrayList(); Statement st = null; try { @@ -580,7 +580,7 @@ protected List getBookmarks(Connection connection, Metadata private List getDownloads(Connection connection, Metadata metadata, ParseContext context) throws SQLException, JsonParseException, JsonMappingException, IOException { - List downloads = new LinkedList(); + List downloads = new ArrayList(); Statement st = null; try { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/safari/SafariPlistParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/safari/SafariPlistParser.java index 530e0a3566..430751b530 100755 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/safari/SafariPlistParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/safari/SafariPlistParser.java @@ -5,7 +5,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -342,7 +342,7 @@ private void parseSafariResumedHistory(InputStream stream, ContentHandler handle } protected List getResumedHistory(InputStream is) throws Exception { - List resumedHistory = new LinkedList(); + List resumedHistory = new ArrayList(); NSDictionary rootDict = (NSDictionary) PropertyListParser.parse(is); NSObject[] parameters = ((NSArray) rootDict.objectForKey("WebHistoryDates")).getArray(); @@ -371,7 +371,7 @@ protected List getResumedHistory(InputStream is) throws Exce } // protected List getHistory(InputStream is) throws Exception { - // List history = new LinkedList(); + // List history = new ArrayList(); // // NSDictionary rootDict = (NSDictionary) PropertyListParser.parse(is); // NSObject[] parameters = ((NSArray) @@ -400,7 +400,7 @@ protected List getResumedHistory(InputStream is) throws Exce // } private List getDownloads(InputStream is) throws Exception { - List downloads = new LinkedList<>(); + List downloads = new ArrayList<>(); NSDictionary rootDict = (NSDictionary) PropertyListParser.parse(is); NSObject[] parameters = ((NSArray) rootDict.objectForKey("DownloadHistory")).getArray(); @@ -555,7 +555,7 @@ private void parseChildren(List bookmarks, NSDictionary dict) { } private List getBookmarks(InputStream is) throws Exception { - List bookmarks = new LinkedList(); + List bookmarks = new ArrayList(); NSDictionary rootDict = (NSDictionary) PropertyListParser.parse(is); diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/safari/SafariSqliteParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/safari/SafariSqliteParser.java index 28449a901a..91eb95678f 100755 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/safari/SafariSqliteParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/browsers/safari/SafariSqliteParser.java @@ -9,7 +9,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -229,7 +229,7 @@ private void parseSafariResumedHistory(ContentHandler handler, Metadata metadata protected List getResumedHistory(Connection connection, Metadata metadata, ParseContext context) throws SQLException { - List resumedHistory = new LinkedList(); + List resumedHistory = new ArrayList(); Statement st = null; try { @@ -273,7 +273,7 @@ protected List getResumedHistory(Connection connection, Meta protected List getHistory(Connection connection, Metadata metadata, ParseContext context) throws SQLException { - List history = new LinkedList(); + List history = new ArrayList(); Statement st = null; try { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/discord/json/DiscordAttachment.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/discord/json/DiscordAttachment.java index 4a8ddd35a6..158d9724a3 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/discord/json/DiscordAttachment.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/discord/json/DiscordAttachment.java @@ -1,7 +1,7 @@ package iped.parsers.discord.json; -import java.util.HashSet; -import java.util.Set; +import java.util.Collections; +import java.util.List; import com.fasterxml.jackson.annotation.JsonProperty; @@ -42,7 +42,7 @@ public class DiscordAttachment { private String mediaHash; - private Set childPornSets = new HashSet<>(); + private List childPornSets; public String getId() { return id; @@ -122,15 +122,11 @@ public String getMediaHash() { public void setMediaHash(String mediaHash) { this.mediaHash = mediaHash; - childPornSets.addAll(ChildPornHashLookup.lookupHash(mediaHash)); + childPornSets = ChildPornHashLookup.lookupHashAndMerge(mediaHash, childPornSets); } - public Set getChildPornSets() { - return childPornSets; - } - - public void setChildPornSets(Set childPornSets) { - this.childPornSets = childPornSets; + public List getChildPornSets() { + return childPornSets == null ? Collections.emptyList() : childPornSets; } @Override diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/emule/KnownMetParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/emule/KnownMetParser.java index 3dfd0326b5..ebd261d792 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/emule/KnownMetParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/emule/KnownMetParser.java @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.TimeZone; @@ -199,13 +198,12 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, cells.add(e.getName()); String hash = e.getHash(); metadata.add(ExtraProperties.SHARED_HASHES, hash); - HashSet hashSets = new HashSet<>(); - hashSets.addAll(ChildPornHashLookup.lookupHash(EDONKEY, hash)); + List hashSets = ChildPornHashLookup.lookupHash(EDONKEY, hash); item = searchItemInCase(searcher, EDONKEY, e.getHash()); if(item != null) { - hashSets.addAll(ChildPornHashLookup.lookupHash(item.getHash())); + hashSets = ChildPornHashLookup.lookupHashAndMerge(EDONKEY, hash, hashSets); } - if (!hashSets.isEmpty()) { + if (hashSets != null && !hashSets.isEmpty()) { hashDBHits++; trClass = "rr"; //$NON-NLS-1$ } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/emule/PartMetParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/emule/PartMetParser.java index fe3235c715..94b5ed9888 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/emule/PartMetParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/emule/PartMetParser.java @@ -8,7 +8,7 @@ import java.text.SimpleDateFormat; import java.util.Collections; import java.util.Date; -import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.TimeZone; @@ -115,12 +115,11 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, xhtml.startElement("table", "class", "d"); int hashDBHits = 0; - HashSet hashSets = new HashSet(); - hashSets.addAll(ChildPornHashLookup.lookupHash(KnownMetParser.EDONKEY, e.getHash())); + List hashSets = ChildPornHashLookup.lookupHash(KnownMetParser.EDONKEY, e.getHash()); IItemReader item = KnownMetParser.searchItemInCase(searcher, KnownMetParser.EDONKEY, e.getHash()); if (item != null) - hashSets.addAll(ChildPornHashLookup.lookupHash(item.getHash())); - if (!hashSets.isEmpty()) + hashSets = ChildPornHashLookup.lookupHashAndMerge(item.getHash(), hashSets); + if (hashSets != null && !hashSets.isEmpty()) hashDBHits++; AttributesImpl attributes = new AttributesImpl(); diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/jdbc/JDBCTableReader.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/jdbc/JDBCTableReader.java index f1d7c677f2..b46605e4ba 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/jdbc/JDBCTableReader.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/jdbc/JDBCTableReader.java @@ -27,7 +27,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import javax.swing.text.html.HTMLEditorKit.Parser; @@ -159,7 +159,7 @@ public String handleCell(ResultSet results, ResultSetMetaData rsmd, int i, Conte } public List getHeaders() throws IOException { - List headers = new LinkedList(); + List headers = new ArrayList(); // lazy initialization if (results == null) { results = getTableData(); diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/LibraryFile.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/LibraryFile.java index 0b4fe46dfe..7bbbb16277 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/LibraryFile.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/LibraryFile.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashSet; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -65,7 +65,7 @@ public class LibraryFile extends ShareazaEntity { private boolean cachedPreview; private boolean bogus; private final LibraryFolder parentFolder; - private HashSet hashSetHits = new HashSet<>(); + private List hashSetHits; public LibraryFile(LibraryFolder parentFolder) { super("LIBRARY FILE"); //$NON-NLS-1$ @@ -208,14 +208,14 @@ public boolean isShared() { public void printTableRow(XHTMLContentHandler html, String path, IItemSearcher searcher, Map> albunsForFiles) throws SAXException { - hashSetHits.addAll(ChildPornHashLookup.lookupHash(md5)); - hashSetHits.addAll(ChildPornHashLookup.lookupHash(sha1)); + hashSetHits = ChildPornHashLookup.lookupHashAndMerge(md5, hashSetHits); + hashSetHits = ChildPornHashLookup.lookupHashAndMerge(sha1, hashSetHits); AttributesImpl attributes = new AttributesImpl(); if (md5 != null && !md5.isEmpty()) { attributes.addAttribute("", "name", "name", "CDATA", md5.toUpperCase()); } - if (!hashSetHits.isEmpty()) { + if (hashSetHits != null && !hashSetHits.isEmpty()) { attributes.addAttribute("", "class", "class", "CDATA", "r"); } html.startElement("tr", attributes); @@ -260,7 +260,7 @@ private void printTd(XHTMLContentHandler html, IItemSearcher searcher, Object... col++; } html.startElement("td"); //$NON-NLS-1$ - if (!hashSetHits.isEmpty()) { + if (hashSetHits != null && !hashSetHits.isEmpty()) { html.characters(hashSetHits.toString()); } html.endElement("td"); //$NON-NLS-1$ @@ -270,7 +270,7 @@ private void printTd(XHTMLContentHandler html, IItemSearcher searcher, Object... } public boolean isHashDBHit() { - return !hashSetHits.isEmpty(); + return hashSetHits != null && !hashSetHits.isEmpty(); } public long getSize() { @@ -377,8 +377,8 @@ public LibraryFolder getParentFolder() { return parentFolder; } - public HashSet getHashSetHits() { - return hashSetHits; + public List getHashSetHits() { + return hashSetHits == null ? Collections.emptyList() : hashSetHits; } public void setIndex(int index) { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/sqlite/SQLite3DBParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/sqlite/SQLite3DBParser.java index 965de29471..ffbdc34078 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/sqlite/SQLite3DBParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/sqlite/SQLite3DBParser.java @@ -26,7 +26,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -189,7 +189,7 @@ protected String getJDBCClassName() { @Override protected List getTableNames(Connection connection, Metadata metadata, ParseContext context) throws SQLException { - List tableNames = new LinkedList(); + List tableNames = new ArrayList(); Statement st = null; try { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java index eb66fd68c2..894bc86f4a 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Message.java @@ -19,10 +19,9 @@ package iped.parsers.telegram; import java.util.Collection; +import java.util.Collections; import java.util.Date; -import java.util.HashSet; import java.util.List; -import java.util.Set; import dpf.ap.gpinf.interfacetelegram.MessageInterface; import dpf.ap.gpinf.interfacetelegram.PhotoData; @@ -52,7 +51,7 @@ public class Message implements MessageInterface { private long toId = 0; private Double latitude = null; private Double longitude = null; - private Set childPornSets = new HashSet<>(); + private List childPornSets; public long getId() { return id; @@ -68,7 +67,7 @@ public String getMediaHash() { public void setMediaHash(String mediaHash) { this.mediaHash = mediaHash; - childPornSets.addAll(ChildPornHashLookup.lookupHash(mediaHash)); + childPornSets = ChildPornHashLookup.lookupHashAndMerge(mediaHash, childPornSets); } public String getMediaFile() { @@ -248,8 +247,8 @@ public void setLongitude(Double longitude) { this.longitude = longitude; } - public Set getChildPornSets() { - return this.childPornSets; + public List getChildPornSets() { + return childPornSets == null ? Collections.emptyList() : childPornSets; } public void addChildPornSets(Collection sets) { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ChildPornHashLookup.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ChildPornHashLookup.java index 4686e12f12..5e317e6b6f 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ChildPornHashLookup.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/ChildPornHashLookup.java @@ -19,8 +19,17 @@ public static void dispose() { } public static List lookupHash(String mediaHash) { + return lookupHashAndMerge(mediaHash, null); + } + + /** + * Lookup a hash in the database and merge the result with a optionally provided + * list (returned from a previous call to hash lookup). The hash algorithm (md5, + * sha-1 or sha-256) is guessed from the hash length. + */ + public static List lookupHashAndMerge(String mediaHash, List prev) { if (mediaHash == null || mediaHash.isEmpty() || lookupProviders.isEmpty()) - return Collections.EMPTY_LIST; + return prev != null ? prev : Collections.emptyList(); String guessedAlgo = ""; if (mediaHash.length() == 32) guessedAlgo = "md5"; @@ -29,19 +38,37 @@ else if (mediaHash.length() == 40) else if (mediaHash.length() == 64) guessedAlgo = "sha-256"; - return lookupHash(guessedAlgo, mediaHash); + return lookupHashAndMerge(guessedAlgo, mediaHash, prev); } public static List lookupHash(String algorithm, String hash) { - Set hashsets = new HashSet(); + return lookupHashAndMerge(algorithm, hash, null); + } + + /** + * Lookup a hash in the database and merge the result with a optionally provided + * list (returned from a previous call to hash lookup), using a defined + * algorithm. + */ + public static List lookupHashAndMerge(String algorithm, String hash, List prev) { + Set hashsets = null; if (hash != null) { for (LookupProvider provider : lookupProviders) { List sets = provider.lookupHash(algorithm, hash); if (sets != null) { + if (hashsets == null) { + hashsets = new HashSet(); + } hashsets.addAll(sets); } } } + if (hashsets == null || hashsets.isEmpty()) { + return prev != null ? prev : Collections.emptyList(); + } + if (prev != null) { + hashsets.addAll(prev); + } List l = new ArrayList(hashsets); Collections.sort(l); return l; diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/Util.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/Util.java index 3d3a171724..668c5a65d6 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/Util.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/Util.java @@ -25,6 +25,7 @@ import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; +import iped.data.IItem; import iped.data.IItemReader; import iped.parsers.standard.RawStringParser; import iped.parsers.standard.StandardParser; @@ -460,5 +461,27 @@ public static File getFileRenamedToExt(File file, String ext) { return file; } + public static File getFileWithRightExt(IItem item) throws IOException { + File file = item.getTempFile(); + String ext = item.getType(); + boolean isTmpFile = IOUtil.isTemporaryFile(file); + boolean badExt = !ext.isEmpty() && !file.getName().endsWith("." + ext); + if (!isTmpFile && badExt) { + File tmp = File.createTempFile("iped", "." + ext); + tmp.deleteOnExit(); + IOUtil.copyFile(file, tmp); + return tmp; + } else { + if (isTmpFile) { + file.deleteOnExit(); + if (badExt) { + file = Util.getFileRenamedToExt(file, ext); + } + } else { + file.setReadOnly(); + } + return file; + } + } } diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java index 1a11c761cc..1de572b0d1 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java @@ -47,7 +47,6 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -262,7 +261,7 @@ protected List extractChatList() throws WAExtractorException { private List undeleteChats(SQLiteUndeleteTable undeleteChatListTable, SQLiteUndeleteTable undeleteChatTable, SQLiteUndeleteTable undeleteJIDTable, WAContactsDirectory contacts) { - List result = new LinkedList<>(); + List result = new ArrayList<>(); if (undeleteChatListTable != null && undeleteChatListTable.getTableRows() != null && !undeleteChatListTable.getTableRows().isEmpty()) { // this is the case of a database with the table "chat_list" diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java index 64411d478f..a236c27c9c 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorIOS.java @@ -45,7 +45,6 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -554,7 +553,7 @@ private void decodeReceiptInfo(Message m, byte[] receiptInfo) { } private List undeleteChats(SQLiteUndeleteTable undeleteChatsSessions, WAContactsDirectory contacts) { - List result = new LinkedList<>(); + List result = new ArrayList<>(); if (undeleteChatsSessions != null && !undeleteChatsSessions.getTableRows().isEmpty()) { for (SqliteRow row : undeleteChatsSessions.getTableRows()) { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java index e357d74293..aeba963210 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/Message.java @@ -8,11 +8,9 @@ import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Base64; -import java.util.Collection; +import java.util.Collections; import java.util.Date; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -63,10 +61,10 @@ public class Message { private int mediaDuration; private MessageStatus messageStatus; private String recoveredFrom = null; - private Set childPornSets = new HashSet<>(); + private List childPornSets; private IItemReader mediaItem = null; private String mediaQuery = null; - private List addOns = new ArrayList<>(); + private List addOns; static { try { @@ -99,7 +97,6 @@ public static synchronized void closeStaticResources() throws IOException { public Message() { messageType = MessageType.TEXT_MESSAGE; - vcards = new ArrayList<>(); } public long getId() { @@ -219,7 +216,7 @@ public void setMediaHash(String mediaHash, boolean encoded) { } else { this.mediaHash = mediaHash; } - childPornSets.addAll(ChildPornHashLookup.lookupHash(this.mediaHash)); + childPornSets = ChildPornHashLookup.lookupHashAndMerge(this.mediaHash, childPornSets); } public byte[] getThumbData() { @@ -320,7 +317,7 @@ public void setMessageType(MessageType messageType) { } public List getVcards() { - return vcards; + return vcards == null ? Collections.emptyList() : vcards; } public void setVcards(List vcards) { @@ -424,12 +421,12 @@ public void setRecoveredFrom(String recoveredFrom) { this.recoveredFrom = recoveredFrom; } - public Set getChildPornSets() { - return childPornSets; + public List getChildPornSets() { + return childPornSets == null ? Collections.emptyList() : childPornSets; } - public void addChildPornSets(Collection sets) { - this.childPornSets.addAll(sets); + public void lookupAndAddChildPornSets(String hash) { + childPornSets = ChildPornHashLookup.lookupHashAndMerge(hash, childPornSets); } public IItemReader getMediaItem() { @@ -449,11 +446,14 @@ public void setMediaQuery(String mediaQuery) { } public boolean addMessageAddOn(MessageAddOn m) { + if (addOns == null) { + addOns = new ArrayList(1); + } return addOns.add(m); } public List getAddOns() { - return addOns; + return addOns == null ? Collections.emptyList() : addOns; } public String getCallId() { diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java index b8cba4de19..beb1186063 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ReportGenerator.java @@ -17,7 +17,6 @@ import org.apache.commons.text.lookup.StringLookupFactory; import iped.data.IItemReader; -import iped.parsers.util.ChildPornHashLookup; import iped.parsers.util.Messages; import iped.parsers.vcard.VCardParser; import iped.parsers.whatsapp.Message.MessageType; @@ -641,7 +640,7 @@ private synchronized void printMessage(PrintWriter out, Message message, boolean break; } if (mediaItem != null) { - message.addChildPornSets(ChildPornHashLookup.lookupHash(mediaItem.getHash())); + message.lookupAndAddChildPornSets(mediaItem.getHash()); } break; } diff --git a/iped-viewers/iped-viewers-impl/src/main/java/iped/viewers/HtmlLinkViewer.java b/iped-viewers/iped-viewers-impl/src/main/java/iped/viewers/HtmlLinkViewer.java index 4ada0807a2..3aae1b6cdd 100644 --- a/iped-viewers/iped-viewers-impl/src/main/java/iped/viewers/HtmlLinkViewer.java +++ b/iped-viewers/iped-viewers-impl/src/main/java/iped/viewers/HtmlLinkViewer.java @@ -144,10 +144,7 @@ public void open(final String luceneQuery) { } File file = null; try { - file = Util.getFileRenamedToExt(item.getTempFile(), item.getType()); - if (IOUtil.isTemporaryFile(file)) { - file.deleteOnExit(); - } + file = Util.getFileWithRightExt(item); } catch (IOException e1) { e1.printStackTrace(); } diff --git a/pom.xml b/pom.xml index 1e2326cd42..833bdc0ea3 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ 3.34.0 1.1.2 1.0.9-SNAPSHOT - 3.9.4 + 3.10.1