Skip to content

Commit

Permalink
Merge branch 'master' of github.com:sepinf-inc/IPED into minio_ufdr
Browse files Browse the repository at this point in the history
  • Loading branch information
@hauck-jvsh
hauck-jvsh committed Aug 15, 2024
2 parents d933265 + 69089c1 commit 9f2e0f2
Show file tree
Hide file tree
Showing 12 changed files with 97 additions and 88 deletions.
2 changes: 2 additions & 0 deletions iped-api/src/main/java/iped/properties/ExtraProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ public class ExtraProperties {

public static final String GROUP_ID = "GroupID";

public static final String IS_GROUP_MESSAGE = "isGroupMessage";

public static final String MESSAGE_BODY = MESSAGE_PREFIX + "Body"; //$NON-NLS-1$

public static final String MESSAGE_IS_ATTACHMENT = Message.MESSAGE_PREFIX + "IsEmailAttachment"; //$NON-NLS-1$
Expand Down
42 changes: 17 additions & 25 deletions iped-engine/src/main/java/iped/engine/datasource/UfedXmlReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.Random;
Expand Down Expand Up @@ -1185,43 +1184,36 @@ private void processItem(Item item) throws SAXException {
}
}

private HashMap<String, String[]> toCache = new LinkedHashMap<String, String[]>(16, 0.75f, true) {

private static final long serialVersionUID = 1L;

@Override
protected boolean removeEldestEntry(Entry<String, String[]> eldest) {
return this.size() > 1000;
}
};

private Property toProperty = Property.internalText(ExtraProperties.COMMUNICATION_TO);

private void fillMissingInfo(Item item) {
String from = item.getMetadata().get(ExtraProperties.COMMUNICATION_FROM);
String to = item.getMetadata().get(ExtraProperties.COMMUNICATION_TO);
String[] to = item.getMetadata().getValues(ExtraProperties.COMMUNICATION_TO);
boolean fromOwner = Boolean.valueOf(item.getMetadata().get(ExtraProperties.UFED_META_PREFIX + "fromOwner"));
if (to == null) {
if (to == null || to.length != 1) {
if (item.getMediaType() != null
&& MediaTypes.isInstanceOf(item.getMediaType(), MediaTypes.UFED_MESSAGE_MIME)) {
// we have seen ufed messages without parent chat
if (itemSeq.size() == 0)
return;
IItem parentChat = itemSeq.get(itemSeq.size() - 1);
String[] parties = parentChat.getMetadata()
.getValues(ExtraProperties.UFED_META_PREFIX + "Participants");
ArrayList<String> toList = new ArrayList<>();
for (String party : parties) {
if ((from != null && !party.equals(from)) || (fromOwner && !ownerParties.contains(party)))
toList.add(party);
List<String> toList = new ArrayList<>();
if (to != null && to.length > 0) {
toList = Arrays.asList(to);
} else {
String[] parties = parentChat.getMetadata().getValues(ExtraProperties.UFED_META_PREFIX + "Participants");
for (String party : parties) {
if ((from != null && !party.equals(from)) || (fromOwner && !ownerParties.contains(party)))
toList.add(party);
}
}
String key = toList.toString();
String[] val = toCache.get(key);
if (val == null) {
val = toList.toArray(new String[toList.size()]);
toCache.put(key, val);
if (toList.size() == 1) {
item.getMetadata().set(toProperty, toList.get(0));
} else if (toList.size() > 1) {
item.getMetadata().set(toProperty, parentChat.getName());
item.getMetadata().set(ExtraProperties.IS_GROUP_MESSAGE, "true");
}
item.getMetadata().set(toProperty, val);

}
}
if (!msisdns.isEmpty() && (MediaTypes.UFED_CALL_MIME.equals(item.getMediaType())
Expand Down
11 changes: 10 additions & 1 deletion iped-engine/src/main/java/iped/engine/graph/GraphTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,10 @@ private NodeValues getPhoneNodeValues(String value) {
return null;
}

private NodeValues getGroupNodeValues(String value) {
return new NodeValues(DynLabel.label(GraphConfiguration.CONTACT_GROUP_LABEL), BasicProps.NAME, value.trim().toLowerCase());
}

private NodeValues getGenericNodeValues(String value) {
return new NodeValues(DynLabel.label("GENERIC"), "entity", value.trim().toLowerCase());
}
Expand Down Expand Up @@ -469,7 +473,12 @@ private void processCommunicationMetadata(IItem evidence) throws IOException {
recipients.addAll(Arrays.asList(metadata.getValues(Message.MESSAGE_BCC)));

for (String recipient : recipients) {
NodeValues nv2 = getNodeValues(recipient, evidence.getMetadata(), detectPhones);
NodeValues nv2;
if (Boolean.valueOf(metadata.get(ExtraProperties.IS_GROUP_MESSAGE))) {
nv2 = getGroupNodeValues(recipient);
} else {
nv2 = getNodeValues(recipient, metadata, detectPhones);
}
graphFileWriter.writeNode(nv2.label, nv2.propertyName, nv2.propertyValue, nv2.props);
graphFileWriter.writeRelationship(nv1.label, nv1.propertyName, nv1.propertyValue, nv2.label,
nv2.propertyName, nv2.propertyValue, relationshipType, relProps);
Expand Down
20 changes: 11 additions & 9 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/discord/DiscordParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@

import iped.data.IItemReader;
import iped.parsers.browsers.chrome.CacheIndexParser;
import iped.parsers.discord.cache.CacheAddr.InputStreamNotAvailable;
import iped.parsers.discord.cache.Index;
import iped.parsers.discord.json.DiscordAttachment;
import iped.parsers.discord.json.DiscordAuthor;
Expand Down Expand Up @@ -324,15 +323,18 @@ private void extractMessages(String chatName, List<DiscordRoot> discordRoot, Con
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, d.getAuthor().getFullUsername());
meta.set(ExtraProperties.DECODED_DATA, Boolean.TRUE.toString());

// Add "Message-TO" field
for (String participant : participants) {
if (participant.length() <= 1) {
// In cases where only one participant sends messages, it is not possible to
// determine the participants as only the participant list of the calls are
// cached.
} else if (!participant.equals(d.getAuthor().getFullUsername())) {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, participant);
// Add "Message-TO" field.
// In cases where only one participant sends messages, it is not possible to
// determine the participants as only the participant list of the calls are cached.
if (participants.size() <= 2) {
for (String participant : participants) {
if (!participant.equals(d.getAuthor().getFullUsername())) {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, participant);
}
}
} else {
meta.set(org.apache.tika.metadata.Message.MESSAGE_TO, chatName);
meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true");
}

for (DiscordAttachment da : d.getAttachments()) {
Expand Down
10 changes: 2 additions & 8 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -253,13 +252,8 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata,
tMetadata.set(ExtraProperties.MESSAGE_DATE, t.getStart());
tMetadata.set(ExtraProperties.MESSAGE_BODY, name);
tMetadata.set(Metadata.MESSAGE_FROM, formatSkypeName(contactMap, t.getFrom()));
if (t.getType() == 3 && t.getConversation().getParticipantes() != null) {
for (Iterator<String> iterator = t.getConversation().getParticipantes().iterator(); iterator
.hasNext();) {
String recip = formatSkypeName(contactMap, iterator.next());
if (recip != null)
tMetadata.add(Metadata.MESSAGE_TO, recip);
}
if (t.getType() == 3 && t.getConversation().getParticipantes() != null && t.getConversation().getParticipantes().size() > 1) {
tMetadata.set(Metadata.MESSAGE_TO, t.getConversation().getTitle());
} else {
tMetadata.set(Metadata.MESSAGE_TO, formatSkypeName(contactMap, t.getTo()));
}
Expand Down
2 changes: 1 addition & 1 deletion iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/Extractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ protected List<Chat> extractChatList() throws Exception {
ChatGroup group = new ChatGroup(chatId, cont, chatName);

Map<String, Object> m = androidDecoder.getAlltMetadata();
if ("true".equals(m.get("broadcast")) || "true".equals(m.get("megagroup"))) {
if ("true".equals(m.get("broadcast"))) {
group.setGroup(false);
group.setChannel(true);
}
Expand Down
8 changes: 4 additions & 4 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,10 +266,10 @@ private void extractMessages(String chatName, List<Message> messages, Contact ac
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, m.getFrom().toString());
if (m.getChat().isGroupOrChannel()) {
ChatGroup groupChat = (ChatGroup) m.getChat();
for (Long id : groupChat.getMembers()) {
if (id != m.getFrom().getId())
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, e.getContact(id).toString());
}
String to = groupChat.isGroup() ? "Group " : "Channel ";
to += groupChat.getName() + " (id:" + groupChat.getId() + ")";
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to);
meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true");
}
if (meta.get(org.apache.tika.metadata.Message.MESSAGE_TO) == null) {
if (m.getToId() != 0) {
Expand Down
17 changes: 9 additions & 8 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,13 +331,14 @@ private ThreemaAccount getUserAccount(IItemSearcher searcher) {
return account;
}

private void fillGroupRecipients(Metadata meta, Chat c, String from) {
for (ThreemaContact member : c.getParticipants()) {
String gmb = member.getFullId();
if (!gmb.equals(from)) {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, gmb);
}
private void fillGroupRecipients(Metadata meta, Chat c) {
String to = "Group";
if (c.getSubject() != null) {
to += " " + c.getSubject().strip();
}
to += " (id:" + c.getId() + ")";
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to);
meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true");
}

private void extractMessages(String chatName, Chat c, List<Message> messages, ThreemaAccount account, int parentVirtualId, ContentHandler handler, EmbeddedDocumentExtractor extractor) throws SAXException, IOException {
Expand All @@ -360,14 +361,14 @@ private void extractMessages(String chatName, Chat c, List<Message> messages, Th
if (m.isFromMe()) {
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, local);
if (c.isGroupChat()) {
fillGroupRecipients(meta, c, local);
fillGroupRecipients(meta, c);
} else {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, remote);
}
} else {
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, remote);
if (c.isGroupChat()) {
fillGroupRecipients(meta, c, remote);
fillGroupRecipients(meta, c);
} else {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, local);
}
Expand Down
40 changes: 24 additions & 16 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/util/MetadataUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.apache.commons.lang3.StringUtils;
import org.apache.tika.metadata.IPTC;
import org.apache.tika.metadata.Message;
import org.apache.tika.metadata.Metadata;
Expand Down Expand Up @@ -438,35 +439,42 @@ private static void removeDuplicateKeys(Metadata metadata) {
}

private static void normalizeGPSMeta(Metadata metadata) {
String lat = metadata.get(Metadata.LATITUDE);
if (lat == null)
lat = metadata.get(ExtraProperties.UFED_META_PREFIX + "Latitude");
String lon = metadata.get(Metadata.LONGITUDE);
if (lon == null)
lon = metadata.get(ExtraProperties.UFED_META_PREFIX + "Longitude");
boolean invalid = lat == null || lon == null;
if (!invalid) {
String lat = StringUtils.firstNonBlank(metadata.get(Metadata.LATITUDE),
metadata.get(ExtraProperties.UFED_META_PREFIX + "Latitude"),
metadata.get(ExtraProperties.UFED_META_PREFIX + "Associated Location Latitude"));

String lon = StringUtils.firstNonBlank(metadata.get(Metadata.LONGITUDE),
metadata.get(ExtraProperties.UFED_META_PREFIX + "Longitude"),
metadata.get(ExtraProperties.UFED_META_PREFIX + "Associated Location Longitude"));

boolean valid = StringUtils.isNoneBlank(lat, lon);
if (valid) {
lat = lat.replace(',', '.');
lon = lon.replace(',', '.');
try {
Float lati = Float.valueOf(lat);
Float longit = Float.valueOf(lon);
if ((lati < -90 || lati > 90 || longit < -180 || longit > 180 || Float.isNaN(lati)
|| Float.isNaN(longit)) || (lati == 0.0 && longit == 0.0)) {
invalid = true;
valid = false;
}
} catch (NumberFormatException e) {
invalid = true;
valid = false;
}
}
if (!invalid) {
if (valid) {
metadata.add(ExtraProperties.LOCATIONS, lat + ";" + lon);

// always remove these, if valid, they were stored above
metadata.remove(Metadata.LATITUDE.getName());
metadata.remove(Metadata.LONGITUDE.getName());
metadata.remove(ExtraProperties.UFED_META_PREFIX + "Latitude");
metadata.remove(ExtraProperties.UFED_META_PREFIX + "Longitude");
metadata.remove(ExtraProperties.UFED_META_PREFIX + "Associated Location Latitude");
metadata.remove(ExtraProperties.UFED_META_PREFIX + "Associated Location Longitude");
} else {
metadata.remove(Metadata.ALTITUDE.getName());
}
// always remove these, if valid, they were stored above
metadata.remove(Metadata.LATITUDE.getName());
metadata.remove(Metadata.LONGITUDE.getName());
metadata.remove(ExtraProperties.UFED_META_PREFIX + "Latitude");
metadata.remove(ExtraProperties.UFED_META_PREFIX + "Longitude");
}

private static void removeDuplicateValues(Metadata metadata) {
Expand Down
8 changes: 4 additions & 4 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/ExtractorAndroidNew.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,9 +220,9 @@ private void extractCalls(Connection conn, Map<Long, Chat> idToChat) throws SQLE
m.setMessageType(UNKNOWN_VIDEO_CALL);
if (call_result == 5) {
m.setMessageType(VIDEO_CALL);
} else if (call_result == 4) {
m.setMessageType(MISSED_VIDEO_CALL);
} else if (call_result == 2) {
m.setMessageType(MISSED_VIDEO_CALL);
} else if (call_result == 4) {
m.setMessageType(REFUSED_VIDEO_CALL);
} else if (call_result == 3) {
m.setMessageType(UNAVAILABLE_VIDEO_CALL);
Expand All @@ -231,9 +231,9 @@ private void extractCalls(Connection conn, Map<Long, Chat> idToChat) throws SQLE
m.setMessageType(UNKNOWN_VOICE_CALL);
if (call_result == 5) {
m.setMessageType(VOICE_CALL);
} else if (call_result == 4) {
m.setMessageType(MISSED_VOICE_CALL);
} else if (call_result == 2) {
m.setMessageType(MISSED_VOICE_CALL);
} else if (call_result == 4) {
m.setMessageType(REFUSED_VOICE_CALL);
} else if (call_result == 3) {
m.setMessageType(UNAVAILABLE_VOICE_CALL);
Expand Down
23 changes: 12 additions & 11 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -889,13 +889,14 @@ private String formatContact(WAContact contact, Map<String, String> cache) {
return result;
}

private void fillGroupRecipients(Metadata meta, Chat c, String from, Map<String, String> cache) {
for (WAContact member : c.getGroupMembers()) {
String gmb = formatContact(member, cache);
if (!gmb.equals(from)) {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, gmb);
}
}
private void fillGroupRecipients(Metadata meta, Chat c) {
String to = c.isChannelChat() ? "Channel " : "Group ";
if (c.getSubject() != null) {
to += c.getSubject().strip();
}
to += " (id:" + c.getId() + ")";
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, to);
meta.set(ExtraProperties.IS_GROUP_MESSAGE, "true");
}

private void extractMessages(String chatName, Chat c, List<Message> messages, WAAccount account,
Expand Down Expand Up @@ -926,15 +927,15 @@ private void extractMessages(String chatName, Chat c, List<Message> messages, WA
}
if (m.isFromMe()) {
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, local);
if (c.isGroupChat()) {
fillGroupRecipients(meta, c, local, cache);
if (c.isGroupOrChannelChat()) {
fillGroupRecipients(meta, c);
} else {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, remote);
}
} else {
meta.set(org.apache.tika.metadata.Message.MESSAGE_FROM, remote);
if (c.isGroupChat()) {
fillGroupRecipients(meta, c, remote, cache);
if (c.isGroupOrChannelChat()) {
fillGroupRecipients(meta, c);
} else {
meta.add(org.apache.tika.metadata.Message.MESSAGE_TO, local);
}
Expand Down
2 changes: 1 addition & 1 deletion iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/telegram/TelegramParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public void testTelegramParser() throws IOException, SAXException, TikaException

assertEquals("Telegram (phone: 42777)", telegramtracker.messageto.get(0));
assertEquals("Nickerida (phone: 5561983125151)", telegramtracker.messageto.get(1));
assertEquals("Guilherme Andreúce (phone: 5561986143035)", telegramtracker.messageto.get(150));
assertEquals("Group mixirica e noronhe-se (id:-136232058)", telegramtracker.messageto.get(150));

assertTrue(telegramtracker.messagebody.get(0).contains(
"Código de login: 73632. Não envie esse código para ninguém, nem mesmo que eles digam que são do Telegram!"));
Expand Down

0 comments on commit 9f2e0f2

Please sign in to comment.