Merge pull request #1920 from gfd2020/whatsapp-quote-messages

Whatsapp quote messages

iped-app/resources/localization/iped-parsers-messages.properties

@@ -256,6 +256,14 @@ WhatsAppReport.Contact=Contact
 WhatsAppReport.RecoveredFrom=Recovered from
 WhatsAppReport.FoundInPedoHashDB=Found in Child Porn Alert Hash Database
 WhatsAppReport.Owner=Owner
+WhatsAppReport.Recovered=Recovered
+WhatsAppReport.ReferenceNotFound=Reference not found
+WhatsAppReport.Document=Document
+WhatsAppReport.Photo=Photo
+WhatsAppReport.Audio=Audio
+WhatsAppReport.ChatFragment=This quoted message is located on chat fragment
+WhatsAppReport.ReferenceId=Quoted message reference ID
+WhatsAppReport.Close=Close
 VCardParser.FormattedName=Formatted Name
 VCardParser.Name=Name
 VCardParser.Nickname=Nickname

iped-app/resources/localization/iped-parsers-messages_de_DE.properties

@@ -256,6 +256,14 @@ WhatsAppReport.Contact=Kontakt
 WhatsAppReport.RecoveredFrom=Wiederhergestellt aus
 WhatsAppReport.FoundInPedoHashDB=gefunden in KiPo Hash Database
 WhatsAppReport.Owner=Besitzer
+WhatsAppReport.Recovered=Recovered[TBT]
+WhatsAppReport.ReferenceNotFound=Reference not found[TBT]
+WhatsAppReport.Document=Document[TBT]
+WhatsAppReport.Photo=Photo[TBT]
+WhatsAppReport.Audio=Audio[TBT]
+WhatsAppReport.ChatFragment=This quoted message is located on chat fragment[TBT]
+WhatsAppReport.ReferenceId=Quoted message reference ID[TBT]
+WhatsAppReport.Close=Close[TBT]
 VCardParser.FormattedName=Name formatiert
 VCardParser.Name=Name
 VCardParser.Nickname=Nickname

iped-app/resources/localization/iped-parsers-messages_es_AR.properties

@@ -256,6 +256,14 @@ WhatsAppReport.Contact=Contacto
 WhatsAppReport.RecoveredFrom=Recupardo de
 WhatsAppReport.FoundInPedoHashDB=Encontrado en la base de datos Hash de Alerta de Pornografía Infantil
 WhatsAppReport.Owner=Propietario
+WhatsAppReport.Recovered=Recovered[TBT]
+WhatsAppReport.ReferenceNotFound=Reference not found[TBT]
+WhatsAppReport.Document=Document[TBT]
+WhatsAppReport.Photo=Photo[TBT]
+WhatsAppReport.Audio=Audio[TBT]
+WhatsAppReport.ChatFragment=This quoted message is located on chat fragment[TBT]
+WhatsAppReport.ReferenceId=Quoted message reference ID[TBT]
+WhatsAppReport.Close=Close[TBT]
 VCardParser.FormattedName=Nombre con formato
 VCardParser.Name=Nombre
 VCardParser.Nickname=Sobrenombre

iped-app/resources/localization/iped-parsers-messages_it_IT.properties

@@ -256,6 +256,14 @@ WhatsAppReport.Contact=Contatto
 WhatsAppReport.RecoveredFrom=Recuperato da
 WhatsAppReport.FoundInPedoHashDB=Trovato nel Database di materiale pedopornografico
 WhatsAppReport.Owner=Proprietario
+WhatsAppReport.Recovered=Recovered[TBT]
+WhatsAppReport.ReferenceNotFound=Reference not found[TBT]
+WhatsAppReport.Document=Document[TBT]
+WhatsAppReport.Photo=Photo[TBT]
+WhatsAppReport.Audio=Audio[TBT]
+WhatsAppReport.ChatFragment=This quoted message is located on chat fragment[TBT]
+WhatsAppReport.ReferenceId=Quoted message reference ID[TBT]
+WhatsAppReport.Close=Close[TBT]
 VCardParser.FormattedName=Nome formattato
 VCardParser.Name=Nome
 VCardParser.Nickname=Nickname

iped-app/resources/localization/iped-parsers-messages_pt_BR.properties

@@ -256,6 +256,14 @@ WhatsAppReport.Contact=Contato
 WhatsAppReport.RecoveredFrom=Recuperada de
 WhatsAppReport.FoundInPedoHashDB=Encontrado em base de hashes de pornografia infantil
 WhatsAppReport.Owner=Proprietário
+WhatsAppReport.Recovered=Recuperado
+WhatsAppReport.ReferenceNotFound=Referência não encontrada
+WhatsAppReport.Document=Documento
+WhatsAppReport.Photo=Foto
+WhatsAppReport.Audio=Áudio
+WhatsAppReport.ChatFragment=A mensagem citada está localizada no fragmento do chat
+WhatsAppReport.ReferenceId=ID de referência da mensagem citada
+WhatsAppReport.Close=Fechar
 VCardParser.FormattedName=Nome Formatado
 VCardParser.Name=Nome
 VCardParser.Nickname=Apelido

iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroid.java

@@ -79,6 +79,7 @@ public class ExtractorAndroid extends Extractor {
     private boolean hasMediaDurationCol = false;
     private boolean hasGroupParticiantsTable = true;
     private boolean hasForwardedCol = false;
+    private boolean hasQuoteCol = false;
     private SQLException parsingException = null;
 
     public ExtractorAndroid(String itemPath, File databaseFile, WAContactsDirectory contacts, WAAccount account, boolean recoverDeletedRecords) {
@@ -151,6 +152,7 @@ protected List<Chat> extractChatList() throws WAExtractorException {
                     hasMediaCaptionCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "media_caption"); //$NON-NLS-1$ //$NON-NLS-2$
                     hasMediaDurationCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "media_duration"); //$NON-NLS-1$ //$NON-NLS-2$
                     hasForwardedCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "forwarded"); //$NON-NLS-1$ //$NON-NLS-2$
+                    hasQuoteCol = SQLite3DBParser.checkIfColumnExists(conn, "messages", "quoted_row_id"); 
                     if (!hasChatView) {
                         hasSubjectCol = SQLite3DBParser.checkIfColumnExists(conn, "chat_list", "subject"); //$NON-NLS-1$ //$NON-NLS-2$
                     }
@@ -341,7 +343,34 @@ private List<Message> extractMessages(Connection conn, WAContact remote, boolean
         Set<MessageWrapperForDuplicateRemoval> activeMessages = new HashSet<>();
         Map<Long, Message> activeMessageIds = new HashMap<>();
 
-        String query = getMessagesQuery(hasThumbTable, hasEditVersionCol, hasMediaDurationCol, hasMediaCaptionCol, hasForwardedCol);
+        List<Message> messagesQuotes = new ArrayList<>();
+        HashMap<String, Message> messagesMapUuid = new HashMap<String, Message>();
+        HashMap<Long, Message> messagesMapIdQuote = new HashMap<Long, Message>();
+
+        if (hasQuoteCol){
+            String queryQuote = getMessagesQueryQuote(hasThumbTable, hasEditVersionCol, hasMediaDurationCol, hasMediaCaptionCol);
+
+            try (PreparedStatement stmt = conn.prepareStatement(queryQuote)) {
+                stmt.setFetchSize(1000);
+                stmt.setString(1, id);
+                try (ResultSet rs = stmt.executeQuery()) {
+                    while (rs.next()) {
+                        Message m = createMessageFromDBRow(rs, remote, isGroupChat, false, hasThumbTable, hasEditVersionCol);
+                        messagesQuotes.add(m);   
+                    }
+                }
+            } catch (SQLException e) {
+                if (firstTry || !isSqliteCorruptException(e)) {
+                    // ignore sqlite corrupt error on second try
+                    // to try to recover deleted records instead
+                    throw e;
+                } else if (parsingException == null) {
+                    parsingException = e;
+                }
+            }
+        }
+
+        String query = getMessagesQuery(hasThumbTable, hasEditVersionCol, hasMediaDurationCol, hasMediaCaptionCol, hasForwardedCol, hasQuoteCol);
 
         try (PreparedStatement stmt = conn.prepareStatement(query)) {
             stmt.setFetchSize(1000);
@@ -353,6 +382,10 @@ private List<Message> extractMessages(Connection conn, WAContact remote, boolean
                         activeMessages.add(new MessageWrapperForDuplicateRemoval(m));
                         activeMessageIds.put(m.getId(), m);
                     }
+                    messagesMapIdQuote.put(m.getIdQuote(),m);
+                    if (m.getUuid() != null && !m.getUuid().isEmpty()) {
+                        messagesMapUuid.put(m.getUuid(), m);
+                    }                    
                     messages.add(m);
                 }
             }
@@ -379,6 +412,10 @@ private List<Message> extractMessages(Connection conn, WAContact remote, boolean
                     if (!activeMessages.contains(new MessageWrapperForDuplicateRemoval(m))) { //do not include deleted message if already there
                         if (!activeMessageIds.containsKey(m.getId()) ||
                             !compareMessagesAlmostTheSame(activeMessageIds.get(m.getId()), m)) { //also remove messages with same id that have the same start text (possibly corrupted recovered record)
+                            messagesMapIdQuote.put(m.getIdQuote(),m);
+                            if (m.getUuid() != null && !m.getUuid().isEmpty()) {
+                                messagesMapUuid.put(m.getUuid(), m);
+                            }                            
                             messages.add(m);
                         }
                     }}
@@ -391,6 +428,26 @@ private List<Message> extractMessages(Connection conn, WAContact remote, boolean
 
             Collections.sort(messages, (a, b) -> a.getTimeStamp().compareTo(b.getTimeStamp()));
         }
+
+        //Find quote messages
+        long fakeIds = 2000000000L;
+        for (Message mq: messagesQuotes){
+            Message m = messagesMapIdQuote.get(mq.getId());
+            if (m != null){// Has quote
+                Message original = messagesMapUuid.get(mq.getUuid());//Try to find orginal message in messages
+                if (original != null){// has found original message reference, more complete
+                    m.setMessageQuote(original);
+                }else{// not found original message reference, get info from message_quotes table, less complete
+                    mq.setDeleted(true);
+                    mq.setId(fakeIds--);
+                    m.setMessageQuote(mq);
+                }
+                m.setQuoted(true);
+            }
+        }
+        messagesMapIdQuote.clear();
+        messagesMapUuid.clear();
+
         return messages;
     }
 
@@ -493,6 +550,11 @@ private Message createMessageFromDBRow(ResultSet rs, WAContact remote, boolean i
         m.setDeleted(deleted);
         m.setForwarded(hasForwardedCol && (rs.getInt("forwarded") > 0));
 
+        if (hasQuoteCol){
+            m.setIdQuote(rs.getInt("quoted_row_id"));
+            m.setUuid(rs.getString("key_id"));
+        }
+
         return m;
     }
 
@@ -640,7 +702,7 @@ protected Message.MessageType decodeMessageType(int messageType, int status, Int
     private static final String SELECT_CHAT_LIST_NO_SUBJECT = "SELECT _id as id,key_remote_jid AS contact, " //$NON-NLS-1$
             + " null as subject, 1230768000000 as creation FROM chat_list"; //$NON-NLS-1$
 
-    private static String getMessagesQuery(boolean hasThumbTable, boolean hasEditVersion, boolean hasMediaDuration, boolean hasMediaCaption, boolean hasForwarded) {
+    private static String getMessagesQuery(boolean hasThumbTable, boolean hasEditVersion, boolean hasMediaDuration, boolean hasMediaCaption, boolean hasForwarded, boolean hasQuote) {
         String query;
         if (hasThumbTable) {
             query = SELECT_MESSAGES_THUMBS_TABLE;
@@ -659,6 +721,28 @@ private static String getMessagesQuery(boolean hasThumbTable, boolean hasEditVer
         if (!hasForwarded) {
             query = query.replace("(forwarded & 1) as forwarded, ", "");
         }
+        if (!hasQuote) {
+            query = query.replace("quoted_row_id, messages.key_id, ", "");
+        }
+        return query;
+    }
+
+    private static String getMessagesQueryQuote(boolean hasThumbTable, boolean hasEditVersion, boolean hasMediaDuration, boolean hasMediaCaption) {
+        String query;
+        if (hasThumbTable) {
+            query = SELECT_QUOTES_THUMBS_TABLE;
+        } else {
+            query = SELECT_QUOTES_NO_THUMBS_TABLE;
+        }
+        if (!hasEditVersion) {
+            query = query.replace("edit_version, ", "");
+        }
+        if (!hasMediaDuration) {
+            query = query.replace("media_duration, ", "");
+        }
+        if (!hasMediaCaption) {
+            query = query.replace("media_caption as mediaCaption, ", "null as mediaCaption, ");
+        }
         return query;
     }
 
@@ -676,6 +760,7 @@ private static String getMessagesQuery(boolean hasThumbTable, boolean hasEditVer
             + "media_duration, " //$NON-NLS-1$
             + "media_caption as mediaCaption, " //$NON-NLS-1$
             + "(forwarded & 1) as forwarded, " //$NON-NLS-1$
+            + "quoted_row_id, messages.key_id, " //$NON-NLS-1$
             + "media_hash as mediaHash, raw_data as rawData FROM " //$NON-NLS-1$
             + "messages WHERE remoteId=? and status!=-1 ORDER BY timestamp"; //$NON-NLS-1$
 
@@ -688,7 +773,8 @@ private static String getMessagesQuery(boolean hasThumbTable, boolean hasEditVer
             + "media_duration, " //$NON-NLS-1$
             + "media_caption as mediaCaption, " //$NON-NLS-1$
             + "(forwarded & 1) as forwarded, " //$NON-NLS-1$
-            + "media_hash as mediaHash, thumbnail as thumbData FROM " //$NON-NLS-1$
+            + "quoted_row_id, messages.key_id, " //$NON-NLS-1$          
+            + "media_hash as mediaHash, thumbnail as thumbData FROM " //$NON-NLS-1$          
             + "messages LEFT JOIN message_thumbnails ON (messages.key_id = message_thumbnails.key_id " //$NON-NLS-1$
             + "AND messages.key_remote_jid = message_thumbnails.key_remote_jid " //$NON-NLS-1$
             + "AND messages.key_from_me = message_thumbnails.key_from_me) " //$NON-NLS-1$
@@ -697,6 +783,35 @@ private static String getMessagesQuery(boolean hasThumbTable, boolean hasEditVer
     // to address a field must use ` instead of '
     private static final String SELECT_GROUP_MEMBERS = "select gjid as 'group', jid as member FROM group_participants where `group`=?"; //$NON-NLS-1$
 
+    private static final String SELECT_QUOTES_NO_THUMBS_TABLE = "SELECT mq._id AS id, mq.key_remote_jid "
+            + "as remoteId, mq.remote_resource AS remoteResource, mq.status, mq.data, "
+            + "mq.key_from_me as fromMe, mq.timestamp, mq.media_url as mediaUrl, "
+            + "mq.media_mime_type as mediaMime, mq.media_size as mediaSize, mq.media_name as mediaName, "
+            + "mq.media_wa_type as messageType, null as thumbData, mq.latitude, mq.longitude, "
+            + "mq.edit_version, "
+            + "mq.media_duration, "
+            + "mq.media_caption as mediaCaption, "
+            + "null as forwarded, " //$NON-NLS-1$            
+            + "mq.quoted_row_id, mq.key_id, "         
+            + "mq.media_hash as mediaHash, mq.raw_data as rawData FROM messages_quotes mq "
+            + "WHERE remoteId=? and mq.status!=-1 ORDER BY mq.timestamp";
+
+    private static final String SELECT_QUOTES_THUMBS_TABLE = "SELECT mq._id AS id, mq.key_remote_jid " //$NON-NLS-1$
+            + "as remoteId, mq.remote_resource AS remoteResource, mq.status, mq.data, " //$NON-NLS-1$
+            + "mq.key_from_me as fromMe, mq.timestamp as timestamp, mq.media_url as mediaUrl, " //$NON-NLS-1$
+            + "mq.media_mime_type as mediaMime, mq.media_size as mediaSize, mq.media_name as mediaName, " //$NON-NLS-1$
+            + "mq.media_wa_type as messageType, mq.raw_data as rawData, mq.latitude, mq.longitude, " //$NON-NLS-1$
+            + "mq.edit_version, " //$NON-NLS-1$
+            + "mq.media_duration, " //$NON-NLS-1$
+            + "mq.media_caption as mediaCaption, " //$NON-NLS-1$
+            + "null as forwarded, " //$NON-NLS-1$             
+            + "mq.quoted_row_id, mq.key_id, " //$NON-NLS-1$          
+            + "mq.media_hash as mediaHash, thumbnail as thumbData FROM messages_quotes mq " //$NON-NLS-1$         
+            + "LEFT JOIN message_thumbnails ON (mq.key_id = message_thumbnails.key_id " //$NON-NLS-1$
+            + "AND mq.key_remote_jid = message_thumbnails.key_remote_jid " //$NON-NLS-1$
+            + "AND mq.key_from_me = message_thumbnails.key_from_me) " //$NON-NLS-1$
+            + "WHERE remoteId=? and mq.status!=-1 ORDER BY mq.timestamp"; //$NON-NLS-1$
+
     private static final Map<String, String> MESSAGES_TABLE_COL_MAP = new HashMap<>();
 
     static {