Merge branch 'master' into RegRipperTLN

.github/workflows/maven.yml

@@ -6,7 +6,7 @@ jobs:
 
   build-java11:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
     - uses: actions/checkout@v1
@@ -28,7 +28,7 @@ jobs:
       if: steps.libagdb.outputs.cache-hit != 'true'
       run: | 
             sudo apt install git autoconf automake autopoint libtool pkg-config
-            git clone https://github.com/libyal/libagdb.git && cd libagdb/ && git checkout e858e15
+            git clone https://github.com/libyal/libagdb.git && cd libagdb/ && git checkout 667a782
             sudo ./synclibs.sh && sudo ./autogen.sh && sudo ./configure
 
     - name: Install External Tools
@@ -64,7 +64,7 @@ jobs:
 
   build-java14:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
     - uses: actions/checkout@v1
@@ -87,7 +87,7 @@ jobs:
       if: steps.libagdb.outputs.cache-hit != 'true'
       run: | 
             sudo apt install git autoconf automake autopoint libtool pkg-config
-            git clone https://github.com/libyal/libagdb.git && cd libagdb/ && git checkout e858e15
+            git clone https://github.com/libyal/libagdb.git && cd libagdb/ && git checkout 667a782
             sudo ./synclibs.sh && sudo ./autogen.sh && sudo ./configure
 
     - name: Install External Tools

.gitignore

@@ -4,3 +4,4 @@ nbactions.xml
 /.classpath
 /.project
 .pydevproject
+/.idea/

README.md

@@ -2,7 +2,7 @@
 
 IPED is an open source software that can be used to process and analyze digital evidence, often seized at crime scenes by law enforcement or in a corporate investigation by private examiners.
 
-## Introduction
+## History
 
 IPED - Digital Evidence Processor and Indexer (translated from Portuguese) is a tool implemented in java and originally and still developed by digital forensic experts from Brazilian Federal Police since 2012. Although it was always open source, only in 2019 its code was officially published.
 
@@ -32,13 +32,13 @@ It will generate an snapshot version of IPED in target/release folder.
 
 On Linux you also must build The Sleuthkit and additional dependencies. Please refer to [Linux Section](https://github.com/sepinf-inc/IPED/wiki/Linux)
 
-If you want to contribute to the project, refer to [Contributing](https://github.com/lfcnassif/IPED/wiki/Contributing)
+Contributions are very welcome! Before contributing please refer to [Contributing](https://github.com/lfcnassif/IPED/wiki/Contributing)
 
 ## Features
 
 Some of IPED several features are listed below:
 
-- Supported hashes: md5, sha-1, sha-256, sha-512 and edonkey. PhotoDNA is also available **for law enforcement** (please contact sepinf dot inc dot ditec at pf dot gov dot br)
+- Supported hashes: md5, sha-1, sha-256, sha-512 and edonkey. PhotoDNA is also available **for law enforcement** (please contact iped at pf dot gov dot br)
 - Supported hash sets: NIST NSRL, NIST CAID, ProjectVIC, Interpol ICSE, standard CSV format
 - Fast hash deduplication 
 - Signature analysis
@@ -76,3 +76,35 @@ Some of IPED several features are listed below:
 - Web API for searching remote cases, get file metadata, raw content, decoded text, thumbnails and posting bookmarks
 - Creation of bookmarks/tags for interesting data
 - HTML, CSV reports and portable cases with tagged data
+
+## Screenshots
+
+Processing:
+![image](https://github.com/sepinf-inc/IPED/assets/7276994/bf29b44a-a924-4c65-845c-6282a4b91861)
+
+Analysis:
+![image](https://github.com/sepinf-inc/IPED/assets/7276994/5fca2b65-6763-4bc1-9284-604c8b325d54)
+
+Data Carving & Video Thumbnails:
+![image](https://github.com/sepinf-inc/IPED/assets/7276994/4d908fe5-6cb1-443b-96fa-d937fa1d2e2d)
+
+Regex Results:
+![image](https://github.com/sepinf-inc/IPED/assets/7276994/db34adc7-d7b9-4b56-8a35-99e095380d0b)
+
+Map:
+![image](https://github.com/sepinf-inc/IPED/assets/7276994/279b9280-3a72-484a-8aed-e4d015df196f)
+
+Communication links:
+![image](https://github.com/sepinf-inc/IPED/assets/7276994/8b164948-fa36-47b8-a249-f64547a36b28)
+
+Face search:
+![image](https://github.com/sepinf-inc/IPED/assets/7276994/55ceb13c-dc21-40cd-a6e2-a6e3d6ed49a6)
+
+Audio Transcription:
+![image](https://github.com/sepinf-inc/IPED/assets/7276994/ebded2ad-f88d-43c8-9699-66e498c9939c)
+
+Timeline:
+![image](https://github.com/sepinf-inc/IPED/assets/7276994/011657e3-8ff2-4105-b3c2-116980772fc0)
+
+Time chart:
+![image](https://github.com/sepinf-inc/IPED/assets/7276994/81df1c18-361d-49f1-b755-36520437803a)

ReleaseNotes.txt

@@ -1,3 +1,132 @@
+05/04/2024: IPED-4.1.6:
+Optimizations:
+#455:  Optimization of UFDR reader memory usage (@lfcnassif)
+#2084: Optimize expansion of certain RAR files (@lfcnassif)
+Fixes:
+#1932: Imagemagick portable missing vcomp140.dll dependency [4.1.5 regression] (@lfcnassif)
+#1940: Sometimes items are not included in their bookmarks, when generating a report from a multicase (@wladimirleite)
+#2037: Bookmarks may be lost in multicase reports (@wladimirleite)
+#1975: Processed files in mounted folders with incorrect extension being renamed when opened externally (@lfcnassif)
+#439:  Show warning in GUI for each evidence processed unsuccessfully (@lfcnassif)
+#2092: Incorrect directory tree when parsing partial RAR files (@lfcnassif)
+#1977: Telegram parser duplicating messages (@hauck-jvsh)
+#1921: WhatsApp recovered media messages may become duplicated (iOS only) (@wladimirleite)
+#2089: Failing to process Cellebrite XML Reports (@fmpfeifer, @lfcnassif)
+#2038: Aborting OutOfMemoryError caused by too many results from ItemSearcher called from UFEDChatParser (@lfcnassif, @wladimirleite)
+#2099: Non VMDK file being detected as VMDK causing processing to abort (@fsicoli)
+#2107: Aborting "IllegalArgumentException: DocValuesField "parentIds" is too large, must be <= 32766" caused by very long file name in a UFDR (@wladimirleite)
+#2110: Report generation aborting because of inconsistent data types while indexing a property (@wladimirleite, @lfcnassif)
+#2145: Rare NullPointerException from VideoThumbTask when extracting frames as subitems (@lfcnassif)
+#2141: lastId incorrectly computed after some evidence is removed from case (@lfcnassif)
+#1947: NullPointerException in LanguageDetectTask (@wladimirleite)
+#2077: DecoderException: Odd number of characters from ExportCSVTask if resuming processing (@lfcnassif)
+#1942: Audios not retried and skipped if specific errors happen in remote transcription service (@lfcnassif)
+#1945: Makes Wav2Vec2 transcription robust to new versions of python libraries (@lfcnassif)
+#2102: Weird line wraps when file names contain emojis (@wladimirleite)
+#1989: Parsing exception when searching for chat attachments (@lfcnassif)
+#2051: Stop condition for iped.parsers.whatsapp.Message.setThumbData() recursion (@aberenguel, @lfcnassif)
+#2024: Geopoints are being indexed wrongly inside opensearch (@hauck-jvsh)
+#1938: Fix PDF thumbnail when page is rotated (@wladimirleite)
+#1740: Clear selection not correctly updated on Map (@patrickdalla)
+#1929: Timeline panel needlessly call updateFileListing when clear filters button is pressed (@patrickdalla)
+#1685: Item from UFDR being extracted three times (@lfcnassif, @hauck-jvsh)
+#2041: Using an existing case output folder by accident makes the case unfinished (@wladimirleite)
+#1993: Pressing the "Pause" button during initialization phase cause an exception (@wladimirleite)
+#1988: Avoid searching for regexes in hash values and UUIDs (@wladimirleite)
+#1955: FileNotFoundException might be caused by race condition reading UFDR evidence (@lfcnassif)
+#2064: Negative estimated time to finish UFDR processing (@wladimirleite)
+#1950: Sometimes negative parse times are shown (@wladimirleite)
+#2120: Reduce the automatically set maximum for "-Xmx" from 32GB to 32500MB (@wladimirleite)
+#2008: Minor localization issues on UI (@lfcnassif)
+
+
+05/10/2023: IPED-4.1.5:
+Fixes:
+#1903: RCE vulnerability in libwebp dependency (@tc-wleite, @lfcnassif)
+#1879: Many dates read from UFDR can be decoded using a wrong timezone (@tc-wleite)
+#1898: Discord Parser showing wrong attachment file (@felipecampanini, @lfcnassif)
+#1843: Some deleted chats or messages not being tagged as deleted (@hauck-jvsh)
+#1868: PDF xmp timestamps aren't extracted with timezone info (@patrickdalla)
+#1833: Transcribing audios with more than 2GB on remote service never ends (@hauck-jvsh, @lfcnassif)
+#1880: Error while parsing WhatsApp contacts (@tc-wleite)
+#1840: Fix links to audio and videos in WhatsApp chats, if files are in an input folder (@tc-wleite, @lfcnassif)
+#1836: Broken links in Whatsapp chats when attachments file names contain emojis (@tc-wleite, @gfd2020)
+#1897: Just first regex hit is shown if multiple regex patterns match the same input string (@tc-wleite)
+#1870: NPE in SleuthkitClient when generating report with a virtual disk (@aberenguel, @lfcnassif)
+#1875: ALT+Key to remove from bookmark not working properly with CRTL and SHIFT shortcuts (@tc-wleite)
+#1846: APFS password not set when opening the case on Linux (@aberenguel)
+#1909: Vosk transcription may slow down during large cases processing (@tc-wleite)
+#1842: Improve layout for audio and video tags in whatsapp chats opened in browser (@tc-wleite)
+
+
+16/08/2023: IPED-4.1.4:
+News:
+#1294: Support parsing iCloud backup LZFSE compressed files (@lfcnassif)
+#1525: Support parsing MacOS XXXXX.partial.emlx emails attachments (@FelipeFcosta, @lfcnassif)
+#1798: Support iLBC (Internet Low Bitrate Codec) audios (@tc-wleite)
+#1786: Improve the detection of Matroska files: MKV, MKA and WEBM (@tc-wleite)
+#1815: Improve the detection of WhatsApp iOS account plist file (@lfcnassif)
+#1793: Improve the detection of Apple iWork 13 documents (@lfcnassif)
+#1809: Make VideoThumbTask work on videos in mounted paths longer than 256 chars on Windows (@lfcnassif, @tc-wleite)
+Fixes: 
+#1769: Map renders locations but a blank background [regression 4.1.0] (@lfcnassif)
+#1774: Old WhatsApp databases parsing affected by forwarded message feature [regression 4.1.3] (@hauck-jvsh, @lfcnassif)
+#1791: WhatsApp parser may lose recent messages (@tc-wleite)
+#1765: Aborting IOException from AudioTranscriptTask (@lfcnassif)
+#1801: Never add video: prefix to transcription properties (@lfcnassif)
+#1782: Error opening items inside an E01 from an unmounted READ ONLY Windows network share (@tc-wleite)
+#1762: --remove evidence option fails if graph feature wasn't enabled (@lfcnassif)
+#1757: Infinite loop in IOUtil.ignoreInputStream() causing thread leak and wasting CPU (@hauck-jvsh, @lfcnassif)
+#1814: Corrupted ISO caused an "infinite recursion loop" in SevenZipParser (@tc-wleite, @lfcnassif)
+#1752: TorTcParser timestamp in UTC although not informed (@patrickdalla, @lfcnassif)
+#1795: TorTcParser giving different values on Linux and Windows (@lfcnassif)
+#1750: Absence of JavaFX aborting processing if user clicks on "Preview Case" (@lfcnassif)
+#1806: Mimetype of encrypted Office documents without extension ending with a hyphen (@lfcnassif)
+#1807: Avoid minor NPE while reading AD1 contained file after parsing timeout (@lfcnassif)
+
+
+30/06/2023: IPED-4.1.3:
+News:
+#1287: Flag WhatsApp Forwarded messages (@tc-wleite, @gfd2020)
+#1647: Handle some new and common WhatsApp system messages (@tc-wleite, @lfcnassif)
+#1610: Read WhatsApp owner account information from more sources on Android (@tc-wleite)
+#1559: Decode "Unavailable" WhatsApp audio and video calls (@tc-wleite)
+#1661: Support WhatsApp reactions (@tc-wleite)
+#1655: Improve WhatsApp emojis internal visualization quality (@tc-wleite)
+#1654: Replace WhatsApp HTML preview fonts (@tc-wleite)
+#1636: Emule *.part.met files carving (@hugohmk)
+#1707: Load Timeline chart data just when it becomes visible to decrease memory usage by UI (@patrickdalla)
+#1719: Use Windows trusted certificate store so Map view works through some organization proxies (@patrickdalla)
+#1701: Export items to local case if enableAutomaticExportFiles and enableMinIO are both enabled (@hauck-jvsh)
+#1533: Option to configure OpenSearch/MinIO retries and abort if exhausted (@hauck-jvsh, @lfcnassif)
+#1671: Add support to pass the elasticsearch credentials by ENV vars (@hauck-jvsh)
+#1644: Don't include WhatsApp Business "externally" parsed HTMLs if phoneParsersToUse = "internal" (@tc-wleite)
+#1694: Optimize UFDR evidences opening time through some networks (@lfcnassif, @tc-wleite)
+#1678: Optimize KnownMetCarving to seek instead of skipping huge files (@hugohmk)
+#1724: Makes processing not dependent on stdin again (@lfcnassif)
+#1737: Update localization files for Italian, Spanish and German (@flates, @AburtoArielPM, @mobab-th, @lfcnassif)
+Fixes:
+#1691: Possible incorrect association between WhatsApp accounts and chats when there are multiple accounts if processing on Windows (@tc-wleite)
+#1712: Max heap memory used by Analysis App can be greater than RAM causing UI crashes (@patrickdalla, @lfcnassif)
+#1730: Emule known.met parser missing several entries (@hauck-jvsh, @tc-wleite)
+#1679: WhatsApp parsing timeout can break parsing of other WA databases (@lfcnassif)
+#1663: Processing frozen due to infinite timeouts transcribing huge audios on transcription service (@hauck-jvsh)
+#1664: Problems decoding Cyrillic and other unicode chars from registry files (@lfcnassif)
+#1672: Problems rendering Cyrillic and other Unicode chars extracted from SQLite Tables (@lfcnassif)
+#1668: Aborting IllegalArgumentException: DocValuesField "parentIds" is too large, must be <= 32766 caused by GeofileParser (@patrickdalla)
+#1676: Aborting ArrayIndexOutOfBoundsException from Lucene when creating reports with huge files (@lfcnassif)
+#1686: IllegalArgumentException: Inconsistency of field data structures across documents when generating reports from multicases (@lfcnassif)
+#1726: Artifacts parsing depending on other case items can be incomplete if they are located into splitted embedded disks (@lfcnassif)
+#1660: Aborting FileSystemExeption from deleted Embedded Disks: the file is already being used by another process (@lfcnassif)
+#1684: Metadata with dots not being indexed on opensearch (@hauck-jvsh)
+#1666: Timeline chart draw synchronization problems (@patrickdalla)
+#1589: Error creating report with local wav2vec2 transcription enabled (@lfcnassif)
+#1665: Web service start up not working (@patrickdalla)
+#1677: KnownMetCarveTask might miss some deleted known.met files (@lfcnassif)
+#1723: Processing App may not respond to Ctrl+C on Console (@lfcnassif)
+#1698: Any executable in the case root folder is being copied to the report destination folder (@tc-wleite)
+
+
 17/04/2023: IPED-4.1.2:
 News:
 #1559: Support decoding audio and video calls from android WhatsApp databases v2.22.8+ (@hauck-jvsh, @lfcnassif)

ThirdParty.txt

@@ -48,6 +48,10 @@ Apache TwelveMonkeys
 - Site: https://github.com/haraldk/TwelveMonkeys
 - License: BSD 3 Clause
 
+OFX Client
+- Site: https://github.com/stoicflame/ofx4j
+- License: Apache License, Version 2.0
+
 ----------------------------------------------------
 
 Simple Logging Facade for Java
@@ -175,3 +179,7 @@ Zlib
 JUnRAR
 - Site: https://github.com/edmund-wagner/junrar
 - License: licenses/JUNRAR.txt
+
+Lottie-player
+- Site: https://github.com/LottieFiles/lottie-player/blob/master/LICENSE
+- License: licenses/LOTTIE-PLAYER.txt

iped-api/pom.xml

@@ -11,11 +11,6 @@
     <packaging>jar</packaging>
 
     <dependencies>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>29.0-jre</version>
-        </dependency>
         <dependency>
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-core</artifactId>

iped-api/src/main/java/iped/data/IBookmarks.java

@@ -5,6 +5,7 @@
  */
 package iped.data;
 
+import java.awt.Color;
 import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
@@ -51,6 +52,7 @@ public interface IBookmarks extends Serializable {
 
     int getLastId();
 
+    @Deprecated
     int getTotalItens();
 
     int getTotalChecked();
@@ -91,10 +93,18 @@ public interface IBookmarks extends Serializable {
 
     String getBookmarkComment(int bookmarkId);
 
+    void setBookmarkColor(int bookmarkId, Color color);
+
+    Color getBookmarkColor(int bookmarkId);
+
+    Set<Color> getUsedColors();
+
     void setBookmarkKeyStroke(int bookmarkId, KeyStroke key);
 
     KeyStroke getBookmarkKeyStroke(int bookmarkId);
 
+    void removeBookmarkKeyStroke(int bookmarkId);
+
     int getBookmarkCount(int bookmarkId);
 
     void setInReport(int bookmarkId, boolean inReport);

iped-api/src/main/java/iped/data/IItemReader.java

@@ -7,6 +7,8 @@
 import java.util.HashSet;
 import java.util.Map;
 
+import javax.imageio.stream.ImageInputStream;
+
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.mime.MediaType;
 
@@ -133,6 +135,8 @@ public interface IItemReader extends IStreamSource {
 
     public BufferedInputStream getBufferedInputStream() throws IOException;
 
+    public ImageInputStream getImageInputStream() throws IOException;
+
     /**
      * @return data da última modificação do arquivo
      */

iped-api/src/main/java/iped/data/IMultiBookmarks.java

@@ -5,6 +5,7 @@
  */
 package iped.data;
 
+import java.awt.Color;
 import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
@@ -24,8 +25,11 @@
  */
 public interface IMultiBookmarks extends Serializable {
 
+    @Deprecated
     void addBookmark(List<IItemId> ids, String bookmarkName);
 
+    void addBookmark(Set<IItemId> uniqueSelectedIds, String bookmarkName);
+
     void addToTypedWords(String texto);
 
     void renameBookmark(String oldBookmark, String newBookmark);
@@ -68,8 +72,11 @@ public interface IMultiBookmarks extends Serializable {
 
     void newBookmark(String bookmarkName);
 
+    @Deprecated
     void removeBookmark(List<IItemId> ids, String bookmarkName);
 
+    void removeBookmark(Set<IItemId> uniqueSelectedIds, String bookmarkName);
+
     void saveState();
 
     void saveState(boolean sync);
@@ -80,6 +87,8 @@ public interface IMultiBookmarks extends Serializable {
 
     KeyStroke getBookmarkKeyStroke(String bookmarkName);
 
+    void removeBookmarkKeyStroke(String bookmarkName);
+
     void checkAll();
 
     void setChecked(boolean value, IItemId item);
@@ -90,6 +99,12 @@ public interface IMultiBookmarks extends Serializable {
 
     void setBookmarkComment(String texto, String comment);
 
+    Color getBookmarkColor(String bookmarkName);
+
+    void setBookmarkColor(String bookmarkName, Color color);
+
+    Set<Color> getUsedColors();
+
     boolean isInReport(String bookmark);
 
     void setInReport(String bookmark, boolean checked);