From 16251f8df3c4fc6794a5afd0f4274eee1a724f6c Mon Sep 17 00:00:00 2001 From: Luis Nassif Date: Thu, 23 Nov 2023 10:39:54 -0300 Subject: [PATCH 1/7] '#1857: fill SHARED_HASHES just if shared = true and downloaded > 0 --- .../shareaza/ShareazaDownloadParser.java | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaDownloadParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaDownloadParser.java index 6dba30303f..1d82a6bd83 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaDownloadParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaDownloadParser.java @@ -209,14 +209,14 @@ public void processSDFile(InputStream inputStreamFile, ContentHandler handler, X IItemReader item = null; HashSet hashSets = new HashSet(); + String md5 = null, sha1 = null, edonkey = null; if (sha1Valid != 0) { - String hash = readHashString(buffer, 20); - metadata.add(ExtraProperties.SHARED_HASHES, hash); - addLine(xhtml, "SHA1: " + hash); - hashSets.addAll(ChildPornHashLookup.lookupHash(HASH_SHA1, hash)); + sha1 = readHashString(buffer, 20); + addLine(xhtml, "SHA1: " + sha1); + hashSets.addAll(ChildPornHashLookup.lookupHash(HASH_SHA1, sha1)); if (item == null) { - item = searchItemInCase(searcher, HASH_SHA1, hash); + item = searchItemInCase(searcher, HASH_SHA1, sha1); } } @@ -234,12 +234,11 @@ public void processSDFile(InputStream inputStreamFile, ContentHandler handler, X int md5Valid = readControl4Bytes(buffer); if (md5Valid != 0) { - String hash = readHashString(buffer, 16); - metadata.add(ExtraProperties.SHARED_HASHES, hash); - addLine(xhtml, "MD5: " + hash); - hashSets.addAll(ChildPornHashLookup.lookupHash(HASH_MD5, hash)); + md5 = readHashString(buffer, 16); + addLine(xhtml, "MD5: " + md5); + hashSets.addAll(ChildPornHashLookup.lookupHash(HASH_MD5, md5)); if (item == null) { - item = searchItemInCase(searcher, HASH_MD5, hash); + item = searchItemInCase(searcher, HASH_MD5, md5); } } @@ -251,12 +250,11 @@ public void processSDFile(InputStream inputStreamFile, ContentHandler handler, X int edonkeyValid = readControl4Bytes(buffer); if (edonkeyValid != 0) { - String hash = readHashString(buffer, 16); - metadata.add(ExtraProperties.SHARED_HASHES, hash); - addLine(xhtml, "EDONKEY: " + hash); - hashSets.addAll(ChildPornHashLookup.lookupHash(HASH_EDONKEY, hash)); + edonkey = readHashString(buffer, 16); + addLine(xhtml, "EDONKEY: " + edonkey); + hashSets.addAll(ChildPornHashLookup.lookupHash(HASH_EDONKEY, edonkey)); if (item == null) { - item = searchItemInCase(searcher, HASH_EDONKEY, hash); + item = searchItemInCase(searcher, HASH_EDONKEY, edonkey); } } @@ -418,20 +416,22 @@ public void processSDFile(InputStream inputStreamFile, ContentHandler handler, X int hasFile = read2Bytes(buffer); + long totalDownloaded = 0; + if (hasFile == 1) { sbFile.append("File: " + "\n"); long nTotal = read8Bytes(buffer); long nRemaning = read8Bytes(buffer); int nFragments = read4Bytes(buffer); - long notStart = nTotal - nRemaning; + totalDownloaded = nTotal - nRemaning; sbFile.append(" Total Size: " + nTotal + "\n"); sbFile.append(" Total Remaning: " + nRemaning + "\n"); - sbFile.append(" Total Downloaded: " + notStart + "\n"); + sbFile.append(" Total Downloaded: " + totalDownloaded + "\n"); sbFile.append(" Number of Fragments: " + nFragments + "\n"); - metadata.set(META_PREFIX + "totalDownloaded", Long.toString(notStart)); + metadata.set(META_PREFIX + "totalDownloaded", Long.toString(totalDownloaded)); for (int i = 0; i < nFragments; i++) { long nRangeBegin = read8Bytes(buffer); @@ -701,6 +701,18 @@ public void processSDFile(InputStream inputStreamFile, ContentHandler handler, X addLine(xhtml, "Shared: " + sharedStr); metadata.set(META_PREFIX + "shared", sharedStr); + if (Boolean.valueOf(sharedStr) && totalDownloaded > 0) { + if (md5 != null) { + metadata.add(ExtraProperties.SHARED_HASHES, md5); + } + if (sha1 != null) { + metadata.add(ExtraProperties.SHARED_HASHES, sha1); + } + if (edonkey != null) { + metadata.add(ExtraProperties.SHARED_HASHES, edonkey); + } + } + String serialID = readHashString(buffer, 4); addLine(xhtml, "Serial ID: " + serialID); From 533670a9843808f56136eb5d2de54394ccb2b047 Mon Sep 17 00:00:00 2001 From: Luis Nassif Date: Thu, 23 Nov 2023 11:16:15 -0300 Subject: [PATCH 2/7] '#1857: automatic bookmark for shared files based on Shareaza *.sd files --- .../java/iped/engine/task/P2PBookmarker.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java b/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java index 3dbc4235f4..30a1a2a836 100644 --- a/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java +++ b/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java @@ -3,7 +3,10 @@ import java.awt.Color; import java.io.File; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import org.apache.lucene.document.Document; import org.slf4j.Logger; @@ -19,6 +22,7 @@ import iped.parsers.emule.PartMetParser; import iped.parsers.gdrive.GDriveCloudGraphParser; import iped.parsers.gdrive.GDriveSnapshotParser; +import iped.parsers.shareaza.ShareazaDownloadParser; import iped.parsers.shareaza.ShareazaLibraryDatParser; import iped.parsers.skype.SkypeParser; import iped.parsers.telegram.TelegramParser; @@ -39,7 +43,7 @@ public P2PBookmarker(ICaseData caseData) { } class P2PProgram { - final String hashName; + final List hashNames; final String appName; final Color color; @@ -48,7 +52,11 @@ public P2PProgram(String hashName, String appName) { } public P2PProgram(String hashName, String appName, Color color) { - this.hashName = hashName; + this(Collections.singletonList(hashName), appName, color); + } + + public P2PProgram(List hashNames, String appName, Color color) { + this.hashNames = hashNames; this.appName = appName; this.color = color; } @@ -70,8 +78,13 @@ public void createBookmarksForSharedFiles(File caseDir) { p2pPrograms.put(AresParser.ARES_MIME_TYPE, new P2PProgram(HashTask.HASH.SHA1.toString(), "Ares", new Color(238, 173, 0))); + List shareazaHashes = Arrays.asList(HashTask.HASH.MD5.toString(), HashTask.HASH.SHA1.toString(), HashTask.HASH.EDONKEY.toString()); + p2pPrograms.put(ShareazaLibraryDatParser.LIBRARY_DAT_MIME_TYPE, new P2PProgram(HashTask.HASH.MD5.toString(), "Shareaza", new Color(170, 20, 20))); + + p2pPrograms.put(ShareazaDownloadParser.SHAREAZA_DOWNLOAD_META, + new P2PProgram(shareazaHashes, "Shareaza Downloads", new Color(170, 20, 20))); p2pPrograms.put(WhatsAppParser.WHATSAPP_CHAT.toString(), new P2PProgram(HashTask.HASH.SHA256.toString(), "WhatsApp", new Color(32, 146, 90))); @@ -121,11 +134,15 @@ public void createBookmarksForSharedFiles(File caseDir) { } StringBuilder queryBuilder = new StringBuilder(); queryBuilder.append(IndexItem.LENGTH + ":[3 TO *] AND ("); //$NON-NLS-1$ - if (isHash) - queryBuilder.append(program.hashName + ":("); //$NON-NLS-1$ - queryBuilder.append(items.toString()); - if (isHash) - queryBuilder.append(")"); //$NON-NLS-1$ + if (isHash) { + for (String hash : program.hashNames) { + queryBuilder.append(hash + ":("); //$NON-NLS-1$ + queryBuilder.append(items.toString()); + queryBuilder.append(") "); //$NON-NLS-1$ + } + } else { + queryBuilder.append(items.toString()); + } queryBuilder.append(")"); //$NON-NLS-1$ searcher = new IPEDSearcher(ipedSrc, queryBuilder.toString()); From 61e52c23aa3d789d5768fd34d4a4b0a4372de328 Mon Sep 17 00:00:00 2001 From: Luis Nassif Date: Thu, 23 Nov 2023 11:22:05 -0300 Subject: [PATCH 3/7] '#1857: also create bookmark for Library.dat shares using sha1, edonkey --- .../src/main/java/iped/engine/task/P2PBookmarker.java | 2 +- .../iped/parsers/shareaza/ShareazaLibraryDatParser.java | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java b/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java index 30a1a2a836..0d02b7282d 100644 --- a/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java +++ b/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java @@ -81,7 +81,7 @@ public void createBookmarksForSharedFiles(File caseDir) { List shareazaHashes = Arrays.asList(HashTask.HASH.MD5.toString(), HashTask.HASH.SHA1.toString(), HashTask.HASH.EDONKEY.toString()); p2pPrograms.put(ShareazaLibraryDatParser.LIBRARY_DAT_MIME_TYPE, - new P2PProgram(HashTask.HASH.MD5.toString(), "Shareaza", new Color(170, 20, 20))); + new P2PProgram(shareazaHashes, "Shareaza", new Color(170, 20, 20))); p2pPrograms.put(ShareazaDownloadParser.SHAREAZA_DOWNLOAD_META, new P2PProgram(shareazaHashes, "Shareaza Downloads", new Color(170, 20, 20))); diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaLibraryDatParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaLibraryDatParser.java index cd679f5ed6..7d34a1160d 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaLibraryDatParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaLibraryDatParser.java @@ -170,9 +170,13 @@ private void storeSharedHashes(LibraryFolder folder, Metadata metadata) { private void storeSharedHashes(LibraryFile file, Metadata metadata) { if (file.isShared() && file.getMd5() != null && file.getMd5().length() == 32) { metadata.add(ExtraProperties.SHARED_HASHES, file.getMd5()); - } else if (file.isShared() && file.getSha1() != null && file.getSha1().length() == 40) { + } + if (file.isShared() && file.getSha1() != null && file.getSha1().length() == 40) { metadata.add(ExtraProperties.SHARED_HASHES, file.getSha1()); } + if (file.isShared() && file.getEd2k() != null && file.getEd2k().length() == 32) { + metadata.add(ExtraProperties.SHARED_HASHES, file.getEd2k()); + } } private void storeSharedHashes(AlbumFolder folder, Map indexToFile, Metadata metadata) { From 39e7b9cb423f4d0f7eb5cc58e0f17b73d08d6073 Mon Sep 17 00:00:00 2001 From: Luis Nassif Date: Thu, 23 Nov 2023 11:39:33 -0300 Subject: [PATCH 4/7] '#1857: automatic bookmarks creation for future SHARED_ITEMS prop usage --- iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java b/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java index 0d02b7282d..6ec1cca2cd 100644 --- a/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java +++ b/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java @@ -106,7 +106,7 @@ public void createBookmarksForSharedFiles(File caseDir) { p2pPrograms.put(GDriveSnapshotParser.GDRIVE_SNAPSHOT_REG.toString(), progGDrive); IPEDSource ipedSrc = new IPEDSource(caseDir); - String queryText = ExtraProperties.SHARED_HASHES + ":*"; //$NON-NLS-1$ + String queryText = ExtraProperties.SHARED_HASHES + ":* OR " + ExtraProperties.SHARED_ITEMS + ":*"; IPEDSearcher searcher = new IPEDSearcher(ipedSrc, queryText); try { SearchResult p2pItems = searcher.search(); From 9ad2075ac3d3477d920df0b6e49a2f4452a29d06 Mon Sep 17 00:00:00 2001 From: Luis Nassif Date: Thu, 23 Nov 2023 12:56:24 -0300 Subject: [PATCH 5/7] '#1857: update ShareazaLibraryDatParser tests --- .../shareaza/ShareazaLibraryDatParserTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/shareaza/ShareazaLibraryDatParserTest.java b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/shareaza/ShareazaLibraryDatParserTest.java index 14cfc70f21..35dc9a2629 100644 --- a/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/shareaza/ShareazaLibraryDatParserTest.java +++ b/iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/shareaza/ShareazaLibraryDatParserTest.java @@ -47,10 +47,10 @@ public void testShareazaLibrary1DatParser() throws IOException, SAXException, Ti assertEquals(ShareazaLibraryDatParser.LIBRARY_DAT_MIME_TYPE.toString(), contenttype[0]); assertEquals("228", p2pregistrycount[0]); - assertEquals(153, sharedhashes.length); + assertEquals(459, sharedhashes.length); assertEquals("5ff811cbb56fa306f01aca1890f1a70a", sharedhashes[0]); - assertEquals("07dec8692e10ccfcf765e0807f3d94ce", sharedhashes[1]); - assertEquals("8182ebb4ea93ae9dafaa7cf5b7374bce", sharedhashes[2]); + assertEquals("07dec8692e10ccfcf765e0807f3d94ce", sharedhashes[3]); + assertEquals("8182ebb4ea93ae9dafaa7cf5b7374bce", sharedhashes[6]); } } @@ -82,10 +82,10 @@ public void testShareazaLibrary2DatParser() throws IOException, SAXException, Ti assertEquals(ShareazaLibraryDatParser.LIBRARY_DAT_MIME_TYPE.toString(), contenttype[0]); assertEquals("218", p2pregistrycount[0]); - assertEquals(138, sharedhashes.length); + assertEquals(414, sharedhashes.length); assertEquals("5ff811cbb56fa306f01aca1890f1a70a", sharedhashes[0]); - assertEquals("07dec8692e10ccfcf765e0807f3d94ce", sharedhashes[1]); - assertEquals("025cfa06883c33bcca9b7000e7196718", sharedhashes[2]); + assertEquals("07dec8692e10ccfcf765e0807f3d94ce", sharedhashes[3]); + assertEquals("025cfa06883c33bcca9b7000e7196718", sharedhashes[6]); } } From a41b8ba657baae91671846450558d9c7b1b76beb Mon Sep 17 00:00:00 2001 From: Luis Nassif Date: Thu, 23 Nov 2023 13:12:32 -0300 Subject: [PATCH 6/7] '#1857: removes property causing index data type inconsistency: it should be used just for artificial created subitems --- .../main/java/iped/parsers/shareaza/ShareazaDownloadParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaDownloadParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaDownloadParser.java index 1d82a6bd83..f7cdf45bfd 100644 --- a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaDownloadParser.java +++ b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/shareaza/ShareazaDownloadParser.java @@ -97,7 +97,6 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, processSDFile(stream, handler, xhtml, searcher, metadata, context, item.getPath(), item.getName()); metadata.set(ExtraProperties.P2P_REGISTRY_COUNT, String.valueOf(1)); - metadata.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString()); } catch (TikaException | SAXException | IOException e) { throw e; From d724fe663a55178648bbcb3c795c4a9abf67de18 Mon Sep 17 00:00:00 2001 From: Luis Nassif Date: Thu, 23 Nov 2023 14:56:14 -0300 Subject: [PATCH 7/7] '#1857: minor change to automatic bookmark name --- iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java b/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java index 6ec1cca2cd..b28b49d558 100644 --- a/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java +++ b/iped-engine/src/main/java/iped/engine/task/P2PBookmarker.java @@ -84,7 +84,7 @@ public void createBookmarksForSharedFiles(File caseDir) { new P2PProgram(shareazaHashes, "Shareaza", new Color(170, 20, 20))); p2pPrograms.put(ShareazaDownloadParser.SHAREAZA_DOWNLOAD_META, - new P2PProgram(shareazaHashes, "Shareaza Downloads", new Color(170, 20, 20))); + new P2PProgram(shareazaHashes, "Shareaza SD", new Color(170, 20, 20))); p2pPrograms.put(WhatsAppParser.WHATSAPP_CHAT.toString(), new P2PProgram(HashTask.HASH.SHA256.toString(), "WhatsApp", new Color(32, 146, 90)));