From daae03c558a4f0414c115677b66c14ae7405e06b Mon Sep 17 00:00:00 2001 From: "patrick.pdb" Date: Wed, 21 Feb 2024 14:04:12 -0400 Subject: [PATCH 01/10] '#1864 Initial implementations of modifications to support Time event grouping on Timeline tab. --- .../app/timelinegraph/IpedChartsPanel.java | 55 ++++++++++--- .../app/timelinegraph/TimeEventGroup.java | 61 +++++++++++++++ .../cache/IndexTimeStampCache.java | 21 ++++- .../timelinegraph/cache/TimeIndexedMap.java | 77 ++++++++++++++----- .../timelinegraph/cache/TimeStampCache.java | 9 +++ .../timelinegraph/cache/TimelineCache.java | 17 ++-- .../cache/persistance/CachePersistance.java | 23 ++++-- .../datasets/IpedTimelineDataset.java | 9 ++- .../datasets/IpedTimelineDatasetManager.java | 61 +++++++++++++-- .../java/iped/app/ui/UICaseDataLoader.java | 5 +- .../ui/controls/CheckboxListCellRenderer.java | 64 +++++++++++++++ 11 files changed, 350 insertions(+), 52 deletions(-) create mode 100644 iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java create mode 100644 iped-app/src/main/java/iped/app/ui/controls/CheckboxListCellRenderer.java diff --git a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java index 51b5743634..3db9e45298 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java @@ -10,6 +10,8 @@ import java.awt.Rectangle; import java.awt.event.ComponentEvent; import java.awt.event.ComponentListener; +import java.awt.event.ItemEvent; +import java.awt.event.ItemListener; import java.awt.event.MouseAdapter; import java.awt.event.MouseEvent; import java.awt.font.TextAttribute; @@ -36,9 +38,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Predicate; import javax.swing.DefaultListModel; import javax.swing.ImageIcon; +import javax.swing.JComboBox; import javax.swing.JLabel; import javax.swing.JList; import javax.swing.JPanel; @@ -95,6 +99,7 @@ import iped.app.ui.App; import iped.app.ui.ClearFilterListener; import iped.app.ui.ColumnsManager; +import iped.app.ui.controls.CheckboxListCellRenderer; import iped.app.ui.themes.ThemeManager; import iped.data.IItemId; import iped.engine.search.QueryBuilder; @@ -109,7 +114,11 @@ import iped.viewers.api.ResultSetViewer; import iped.viewers.api.events.RowSorterTableDataChange; -public class IpedChartsPanel extends JPanel implements ResultSetViewer, TableModelListener, ListSelectionListener, IQueryFilterer, ClearFilterListener, ComponentListener { +/** + * @author Patrick Dalla Bernardina + */ +public class IpedChartsPanel extends JPanel implements ResultSetViewer, TableModelListener, ListSelectionListener, + IQueryFilterer, ClearFilterListener, ComponentListener, ItemListener { JTable resultsTable; IMultiSearchResultProvider resultsProvider; GUIProvider guiProvider; @@ -145,6 +154,7 @@ public class IpedChartsPanel extends JPanel implements ResultSetViewer, TableMod IpedChartPanel chartPanel = null; JList legendList = new JList(); JScrollPane listScroller = new JScrollPane(legendList); + JComboBox tegCombo = new JComboBox<>(); IpedStackedXYBarRenderer renderer = null; XYLineAndShapeRenderer highlightsRenderer = new XYLineAndShapeRenderer(); @@ -170,6 +180,7 @@ public class IpedChartsPanel extends JPanel implements ResultSetViewer, TableMod Color fgColor; Color bgColor; + private TimeEventGroup currentTeGroup = TimeEventGroup.BASIC_EVENTS; private static final String resPath = '/' + App.class.getPackageName().replace('.', '/') + '/'; @@ -286,7 +297,18 @@ public void init(JTable resultsTable, IMultiSearchResultProvider resultsProvider splitPane = new IpedSplitPane(); splitPane.setOrientation(JSplitPane.VERTICAL_SPLIT); + chartPanel = new IpedChartPanel(chart, this); + tegCombo.setRenderer(new CheckboxListCellRenderer(new Predicate() { + @Override + public boolean test(TimeEventGroup t) { + return true; + } + })); + chartPanel.add(tegCombo); + + tegCombo.addItemListener(this); + legendListModel = new DefaultListModel(); legendList.setModel(legendListModel); legendList.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); @@ -351,10 +373,6 @@ public void check(MouseEvent e) { domainAxis.setLowerMargin(0.01); domainAxis.setUpperMargin(0.01); combinedPlot.setDomainAxis(domainAxis); - - if (ipedTimelineDatasetManager == null) { - ipedTimelineDatasetManager = new IpedTimelineDatasetManager(this); - } } public String getTimeEventColumnName(String timeEvent) { @@ -399,14 +417,16 @@ public HashMap createDataSets() { if (selectedBookmarks.size() > 0 && chartPanel.getSplitByBookmark()) { for (String bookmark : selectedBookmarks) { - result.put(bookmark, ipedTimelineDatasetManager.getBestDataset(timePeriodClass, bookmark)); + result.put(bookmark, + ipedTimelineDatasetManager.getBestDataset(timePeriodClass, currentTeGroup, bookmark)); } } else if (selectedCategories.size() > 0 && chartPanel.getSplitByCategory()) { for (String category : selectedCategories) { - result.put(category, ipedTimelineDatasetManager.getBestDataset(timePeriodClass, category)); + result.put(category, + ipedTimelineDatasetManager.getBestDataset(timePeriodClass, currentTeGroup, category)); } } else { - result.put("Items", ipedTimelineDatasetManager.getBestDataset(timePeriodClass, null)); + result.put("Items", ipedTimelineDatasetManager.getBestDataset(timePeriodClass, currentTeGroup, null)); } return result; } catch (Exception e) { @@ -1065,13 +1085,19 @@ public void setSyncViewWithTableSelection(boolean syncViewWithTableSelection) { @Override public void checkAll(boolean value) { - // TODO Auto-generated method stub - } @Override public void notifyCaseDataChanged() { + populateEventNames.run(); this.ipedTimelineDatasetManager = new IpedTimelineDatasetManager(this); + + tegCombo.removeAllItems(); + tegCombo.addItem(TimeEventGroup.BASIC_EVENTS); + for (TimeEventGroup teGroup : ipedTimelineDatasetManager.getTimeEventGroupsFromMetadataPrefix()) { + tegCombo.addItem(teGroup); + } + this.dataSetUpdated.set(false); } @@ -1084,5 +1110,12 @@ public void resetZoom() { public static String[] getOrdToEventName() { return ordToEventName; } - + + @Override + public void itemStateChanged(ItemEvent e) { + if (e.getStateChange() == ItemEvent.SELECTED) { + currentTeGroup = (TimeEventGroup) e.getItem(); + refreshChart(true); + } + } } \ No newline at end of file diff --git a/iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java b/iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java new file mode 100644 index 0000000000..ee47b2c4a4 --- /dev/null +++ b/iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java @@ -0,0 +1,61 @@ +package iped.app.timelinegraph; + +import java.util.HashSet; + +/** + * Class that represents a collection of event to use as a filter to graph + * events viewing. Also, the cache will be managed/persisted based on this event + * groups + * + * @author Patrick Dalla Bernardina patrick.dalla@gmail.com + */ +public class TimeEventGroup { + public static final TimeEventGroup ALL_EVENTS = new TimeEventGroup(); + public static final TimeEventGroup BASIC_EVENTS = new TimeEventGroup("BasicProperties"); + + HashSet eventNames = new HashSet(); + + String name; + + private TimeEventGroup() { + this.name = "ALL"; + } + + public TimeEventGroup(String name) { + this.name = name; + } + + public boolean hasEvent(String eventType) { + if (this == ALL_EVENTS) { + return true; + } + return eventNames.contains(eventType); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof TimeEventGroup) { + return name.equals(((TimeEventGroup) obj).name); + } else { + return true; + } + } + + public String getName() { + return name; + } + + public void addEvent(String eventName) { + eventNames.add(eventName); + } + + @Override + public String toString() { + return name; + } +} \ No newline at end of file diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java index b10876b279..11745982f7 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java @@ -21,6 +21,7 @@ import org.roaringbitmap.RoaringBitmap; import iped.app.timelinegraph.IpedChartsPanel; +import iped.app.timelinegraph.TimeEventGroup; import iped.app.timelinegraph.cache.persistance.CachePersistance; import iped.engine.core.Manager; import iped.viewers.api.IMultiSearchResultProvider; @@ -36,6 +37,7 @@ public class IndexTimeStampCache implements TimeStampCache { IMultiSearchResultProvider resultsProvider; IpedChartsPanel ipedChartsPanel; TimeZone timezone; + TimeEventGroup teGroup = TimeEventGroup.ALL_EVENTS;//default TimeEventGroup TimeIndexedMap newCache = new TimeIndexedMap(); @@ -91,7 +93,7 @@ public void run() { int ord = 0; while (ord < cachedEventNames.length) { String eventType = cachedEventNames[ord]; - if (eventType != null && !eventType.isEmpty()) { + if (eventType != null && !eventType.isEmpty() && teGroup.hasEvent(eventType)) { cacheLoaders.add(new EventTimestampCache(ipedChartsPanel, resultsProvider, this, cachedEventNames[ord], ord)); } ord++; @@ -118,7 +120,7 @@ public void run() { newCache = new TimeIndexedMap(); for (Class periodClasses : periodClassesToCache) { - newCache.setIndexFile(periodClasses.getSimpleName(), cp.getBaseDir()); + newCache.setIndexFile(teGroup, periodClasses.getSimpleName(), cp.getBaseDir()); LinkedHashSet times = new LinkedHashSet(); newCache.put(periodClasses.getSimpleName(), times); } @@ -301,4 +303,19 @@ public void add(Class timePeriodClass, Date t, Integer eve selectedCt.addEventEntry(eventInternalOrd, doc); } + @Override + public boolean isFromEventGroup(TimeEventGroup teGroup) { + return this.teGroup.equals(teGroup); + } + + @Override + public void setTimeEventGroup(TimeEventGroup teGroup) { + this.teGroup = teGroup; + } + + @Override + public TimeEventGroup getTimeEventGroup() { + return this.teGroup; + } + } \ No newline at end of file diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java index d500cd6fca..b89eb233cb 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java @@ -14,14 +14,39 @@ import org.apache.pdfbox.io.RandomAccessBufferedFileInputStream; +import iped.app.timelinegraph.TimeEventGroup; import iped.app.timelinegraph.cache.persistance.CachePersistance; +/** + * @author Patrick Dalla Bernardina + */ public class TimeIndexedMap extends HashMap> { - HashMap> upperPeriodIndex = new HashMap>(); - HashMap cacheFiles = new HashMap(); - HashMap monthIndexCacheFiles = new HashMap(); + /* + * Cache persistence will be done based on tuple of TimeEventGroup and Period name + */ + public class Tuple { + TimeEventGroup teGroup; + String periodName; - TimelineCache timelineCache = TimelineCache.get(); + public Tuple(TimeEventGroup teGroup, String periodName) { + this.periodName = periodName; + this.teGroup = teGroup; + } + + @Override + public int hashCode() { + return periodName.hashCode() * teGroup.hashCode(); + } + + @Override + public boolean equals(Object obj) { + return (obj instanceof Tuple) && obj.hashCode() == this.hashCode(); + } + } + + HashMap> upperPeriodIndex = new HashMap>(); + HashMap cacheFiles = new HashMap(); + HashMap monthIndexCacheFiles = new HashMap(); @Override public Set put(String key, Set value) { @@ -30,14 +55,15 @@ public Set put(String key, Set value return result; } - public void setIndexFile(String string, File f) throws IOException { - File cacheFile = new File(new File(f, string), "0"); + public void setIndexFile(TimeEventGroup teGroup, String periodName, File f) throws IOException { + File baseCacheDir = new File(new File(f, teGroup.getName()), periodName); + File cacheFile = new File(baseCacheDir, "0"); if (!cacheFile.exists() || cacheFile.length() == 0) { throw new IOException("File content does not exists:" + f.getName()); } - this.cacheFiles.put(string, cacheFile); - this.monthIndexCacheFiles.put(string, new File(new File(f, string), "1")); + this.cacheFiles.put(new Tuple(teGroup, periodName), cacheFile); + this.monthIndexCacheFiles.put(new Tuple(teGroup, periodName), new File(baseCacheDir, "1")); int committed = 0; try (RandomAccessBufferedFileInputStream lcacheSfis = new RandomAccessBufferedFileInputStream(cacheFile); DataInputStream lcacheDis = new DataInputStream(lcacheSfis)) { @@ -49,11 +75,15 @@ public void setIndexFile(String string, File f) throws IOException { } } + public void setIndexFile(String periodName, File f) throws IOException { + } + Date lastStartDate = null; Date lastEndDate = null; - public RandomAccessBufferedFileInputStream getTmpCacheSfis(String className) throws IOException { - File f = cacheFiles.get(className); + public RandomAccessBufferedFileInputStream getTmpCacheSfis(TimeEventGroup teGroup, String className) + throws IOException { + File f = cacheFiles.get(new Tuple(teGroup, className)); if (f != null) { return new RandomAccessBufferedFileInputStream(f); } else { @@ -61,7 +91,8 @@ public RandomAccessBufferedFileInputStream getTmpCacheSfis(String className) thr } } - public ResultIterator iterator(String className, RandomAccessBufferedFileInputStream lcacheSfis, Date startDate, Date endDate) { + public ResultIterator iterator(TimeEventGroup teGroup, String className, + RandomAccessBufferedFileInputStream lcacheSfis, Date startDate, Date endDate) { try { DataInputStream lcacheDis = new DataInputStream(lcacheSfis); @@ -70,6 +101,7 @@ public ResultIterator iterator(String className, RandomAccessBufferedFileInputSt String timezoneID = lcacheDis.readUTF(); int entries = lcacheDis.readInt(); + TimelineCache timelineCache = TimelineCache.get(teGroup); CacheTimePeriodEntry[] cache = timelineCache.get(className, entries); long startpos = lcacheSfis.getPosition();// position of first entry in cache @@ -78,7 +110,8 @@ public ResultIterator iterator(String className, RandomAccessBufferedFileInputSt // throught month index Long num = null; try { - Entry entry = upperPeriodIndex.get(className).floorEntry(startDate.getTime()); + Entry entry = upperPeriodIndex.get(new Tuple(teGroup, className)) + .floorEntry(startDate.getTime()); if (entry != null) { num = entry.getValue(); } else { @@ -151,7 +184,9 @@ class ResultIterator implements Iterator { // from position in file) private long startPos;// start position in file to iterate - public ResultIterator(long pos, Integer startIndex, TimelineCache timelineCache, RandomAccessBufferedFileInputStream lcacheSfis, DataInputStream lcacheDis, long endDate, String className) { + public ResultIterator(long pos, Integer startIndex, TimelineCache timelineCache, + RandomAccessBufferedFileInputStream lcacheSfis, DataInputStream lcacheDis, long endDate, + String className) { this.startPos = pos; this.startIndex = startIndex; this.lcache = timelineCache.caches.get(className); @@ -241,18 +276,20 @@ public CacheTimePeriodEntry next() { public void createOrLoadUpperPeriodIndex(IndexTimeStampCache indexTimeStampCache) { CachePersistance cp = CachePersistance.getInstance(); if (upperPeriodIndex.size() == 0) { + TimelineCache timelineCache = TimelineCache.get(indexTimeStampCache.getTimeEventGroup()); for (Iterator iterator = indexTimeStampCache.getPeriodClassesToCache().iterator(); iterator.hasNext();) { - String ev = (String) ((Class) iterator.next()).getSimpleName(); - File f = monthIndexCacheFiles.get(ev); + String periodName = (String) ((Class) iterator.next()).getSimpleName(); + File f = monthIndexCacheFiles.get(new Tuple(indexTimeStampCache.getTimeEventGroup(), periodName)); TreeMap datesPos = new TreeMap(); - Map positionsIndexes = timelineCache.getCachesIndexes(ev); + Map positionsIndexes = timelineCache.getCachesIndexes(periodName); try { if (f.exists()) { - cp.loadUpperPeriodIndex(ev, datesPos, positionsIndexes); - upperPeriodIndex.put(ev, datesPos); + cp.loadUpperPeriodIndex(indexTimeStampCache.getTimeEventGroup(), periodName, datesPos, + positionsIndexes); + upperPeriodIndex.put(new Tuple(indexTimeStampCache.getTimeEventGroup(), periodName), datesPos); } } finally { - timelineCache.liberateCachesIndexes(ev); + timelineCache.liberateCachesIndexes(periodName); } } @@ -262,7 +299,7 @@ public void createOrLoadUpperPeriodIndex(IndexTimeStampCache indexTimeStampCache long cacheStartPos = 0; long cacheEndPos = 0; - public HashMap> getMonthIndex() { + public HashMap> getMonthIndex() { return upperPeriodIndex; } diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java index c382920fcc..01597bb767 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java @@ -8,6 +8,9 @@ import org.jfree.data.time.TimePeriod; +import iped.app.timelinegraph.TimeEventGroup; + + public interface TimeStampCache extends Runnable { public void addTimePeriodClassToCache(Class timePeriodClass); @@ -20,4 +23,10 @@ public interface TimeStampCache extends Runnable { public Map> getNewCache(); public TimeZone getCacheTimeZone(); + + public boolean isFromEventGroup(TimeEventGroup teGroup); + + public void setTimeEventGroup(TimeEventGroup teGroup); + + public TimeEventGroup getTimeEventGroup(); } diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimelineCache.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimelineCache.java index 139d0d45ad..e43a78adde 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimelineCache.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimelineCache.java @@ -10,6 +10,8 @@ import java.util.TreeMap; import java.util.concurrent.Semaphore; +import iped.app.timelinegraph.TimeEventGroup; + public class TimelineCache { Date startDate; Date endDate; @@ -25,11 +27,7 @@ public class TimelineCache { // neverUnloadCache.add("Day"); } - static TimelineCache singleton = new TimelineCache(); - - static public TimelineCache get() { - return singleton; - } + static HashMap singletonMap = new HashMap(); private TimelineCache() { } @@ -160,4 +158,13 @@ public HashMap[]> getSoftCaches() { public HashMap getCaches() { return caches; } + + public static TimelineCache get(TimeEventGroup teGroup) { + TimelineCache result = singletonMap.get(teGroup); + if (result == null) { + result = new TimelineCache(); + singletonMap.put(teGroup, result); + } + return result; + } } diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java index 2645e64d23..a2129c1f93 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java @@ -33,6 +33,7 @@ import org.roaringbitmap.RoaringBitmap; import iped.app.timelinegraph.IpedChartsPanel; +import iped.app.timelinegraph.TimeEventGroup; import iped.app.timelinegraph.cache.CacheEventEntry; import iped.app.timelinegraph.cache.CacheTimePeriodEntry; import iped.app.timelinegraph.cache.PersistedArrayList; @@ -160,11 +161,21 @@ public void saveNewCache(TimeStampCache timeStampCache) { e.printStackTrace(); } } + + // set the start dir to persist the cache based on TimeEventGroup + TimeEventGroup teGroup = timeStampCache.getTimeEventGroup(); + File startDir; + if (!teGroup.equals(TimeEventGroup.ALL_EVENTS)) { + startDir = new File(baseDir, teGroup.getName()); + startDir.mkdir(); + } else { + startDir = baseDir; + } TimeIndexedMap newCache = (TimeIndexedMap) timeStampCache.getNewCache(); for (Entry> entry : newCache.entrySet()) { - savePeriodNewCache(timeStampCache, entry.getValue(), new File(baseDir, entry.getKey())); + savePeriodNewCache(teGroup, timeStampCache, entry.getValue(), new File(startDir, entry.getKey())); } } @@ -235,7 +246,8 @@ public long getPosition() { } - private void savePeriodNewCache(TimeStampCache timeStampCache, Set entry, File file) { + private void savePeriodNewCache(TimeEventGroup teGroup, TimeStampCache timeStampCache, + Set entry, File file) { file.mkdirs(); File indexFile = new File(file, "0"); File upperPeriodIndexFile = new File(file, "1"); @@ -257,7 +269,7 @@ private void savePeriodNewCache(TimeStampCache timeStampCache, Set datesPos, Map positionsIndexes) { - File upperPeriodFile = new File(new File(baseDir, ev), "1"); + public void loadUpperPeriodIndex(TimeEventGroup timeEventGroup, String periodName, TreeMap datesPos, + Map positionsIndexes) { + File upperPeriodFile = new File(new File(new File(baseDir,timeEventGroup.getName()), periodName), "1"); try (DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(upperPeriodFile)))) { try { while (true) { diff --git a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java index bb3d104645..b5eaaab37d 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java @@ -65,6 +65,9 @@ import iped.viewers.api.IMultiSearchResultProvider; import iped.viewers.api.IQueryFilterer; +/** + * @author Patrick Dalla Bernardina + */ public class IpedTimelineDataset extends AbstractIntervalXYDataset implements Cloneable, PublicCloneable, IntervalXYDataset, DomainInfo, TimelineDataset, TableXYDataset, XYDomainInfo, AsynchronousDataset { private static final int CTITEMS_PER_THREAD = 500; IMultiSearchResultProvider resultsProvider; @@ -385,9 +388,11 @@ public void caseSearchFilterLoad() throws Exception { cacheWindowEndDate = new Date(ipedChartsPanel.getChartPanel().removeNextFromDatePart(endDate).getTime() - 1); } - try (RandomAccessBufferedFileInputStream sfis = a.getTmpCacheSfis(className)) { + try (RandomAccessBufferedFileInputStream sfis = a.getTmpCacheSfis(cache.getTimeEventGroup(), + className)) { if (sfis != null) { - Iterator it = a.iterator(className, sfis, startDate, endDate); + Iterator it = a.iterator(cache.getTimeEventGroup(), className, sfis, + startDate, endDate); while (it != null && it.hasNext()) { ipedChartsPanel.getIpedTimelineDatasetManager().waitMemory();// method to wait available mem to continue. if (cancelled) { diff --git a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java index aae33c1183..b2fe9351c4 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java @@ -2,6 +2,8 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -22,6 +24,7 @@ import org.jfree.data.xy.AbstractIntervalXYDataset; import iped.app.timelinegraph.IpedChartsPanel; +import iped.app.timelinegraph.TimeEventGroup; import iped.app.timelinegraph.cache.IndexTimeStampCache; import iped.app.timelinegraph.cache.TimeStampCache; import iped.jfextensions.model.Minute; @@ -44,20 +47,39 @@ public IpedTimelineDatasetManager(IpedChartsPanel ipedChartsPanel) { this.ipedChartsPanel = ipedChartsPanel; List> periods = Arrays.asList(Day.class, Hour.class, Year.class, Month.class, Quarter.class, Week.class, Minute.class, Second.class); - + + // includes basic properties time event group first for (Class period : periods) { - TimeStampCache timeStampCache = new IndexTimeStampCache(ipedChartsPanel, ipedChartsPanel.getResultsProvider()); + TimeStampCache timeStampCache = new IndexTimeStampCache(ipedChartsPanel, + ipedChartsPanel.getResultsProvider()); + timeStampCache.setTimeEventGroup(TimeEventGroup.BASIC_EVENTS); timeStampCache.addTimePeriodClassToCache(period); timeStampCaches.add(timeStampCache); } + + for (TimeEventGroup teGroup : getTimeEventGroupsFromMetadataPrefix()) { + if (!teGroup.equals(TimeEventGroup.BASIC_EVENTS)) { + for (Class period : periods) { + TimeStampCache timeStampCache = new IndexTimeStampCache(ipedChartsPanel, + ipedChartsPanel.getResultsProvider()); + timeStampCache.setTimeEventGroup(teGroup); + timeStampCache.addTimePeriodClassToCache(period); + timeStampCaches.add(timeStampCache); + } + } + } + } - public AbstractIntervalXYDataset getBestDataset(Class timePeriodClass, String splitValue) { + public AbstractIntervalXYDataset getBestDataset(Class timePeriodClass, TimeEventGroup teGroup, + String splitValue) { try { for (TimeStampCache timeStampCache : timeStampCaches) { - if (timeStampCache.hasTimePeriodClassToCache(timePeriodClass)) { + if (timeStampCache.hasTimePeriodClassToCache(timePeriodClass) + && timeStampCache.isFromEventGroup(teGroup)) { selectedTimeStampCache = timeStampCache; - return new IpedTimelineDataset(this, ipedChartsPanel.getResultsProvider(), splitValue); + IpedTimelineDataset result = new IpedTimelineDataset(this, ipedChartsPanel.getResultsProvider(), splitValue); + return result; } } return null; @@ -142,4 +164,33 @@ public static long getAvailableMemory() { return Runtime.getRuntime().freeMemory() + Runtime.getRuntime().maxMemory() - Runtime.getRuntime().totalMemory(); } + HashMap groupNames = null; + + public Collection getTimeEventGroupsFromMetadataPrefix() { + if (groupNames == null) { + groupNames = new HashMap(); + String[] cachedEventNames = ipedChartsPanel.getOrdToEventName(); + + for (String eventName : cachedEventNames) { + String groupName; + TimeEventGroup teGroup = TimeEventGroup.BASIC_EVENTS; + groupName = teGroup.getName(); + + // changes the group if there is a prefix + int sepIndex = eventName.indexOf(":"); + if (sepIndex != -1) { + groupName = eventName.substring(0, eventName.indexOf(":")); + teGroup = groupNames.get(groupName); + if (teGroup == null) { + teGroup = new TimeEventGroup(groupName); + groupNames.put(groupName, teGroup); + } + } + + + teGroup.addEvent(eventName); + } + } + return groupNames.values(); + } } diff --git a/iped-app/src/main/java/iped/app/ui/UICaseDataLoader.java b/iped-app/src/main/java/iped/app/ui/UICaseDataLoader.java index d0818a0c9c..1b66e3d7b4 100644 --- a/iped-app/src/main/java/iped/app/ui/UICaseDataLoader.java +++ b/iped-app/src/main/java/iped/app/ui/UICaseDataLoader.java @@ -114,10 +114,10 @@ protected Void doInBackground() { UICaseSearcherFilter pesquisa = new UICaseSearcherFilter(new MatchAllDocsQuery()); pesquisa.execute(); LOGGER.info("Listing all items Finished"); //$NON-NLS-1$ - } else { - App.get().notifyCaseDataChanged(); } + App.get().notifyCaseDataChanged(); + treeModel = new TreeViewModel(); } catch (Throwable e) { @@ -185,6 +185,7 @@ public void done() { ColumnsManager.getInstance().dispose(); App.get().appletListener.updateFileListing(); } + } finally { App.get().dialogBar.setVisible(false); } diff --git a/iped-app/src/main/java/iped/app/ui/controls/CheckboxListCellRenderer.java b/iped-app/src/main/java/iped/app/ui/controls/CheckboxListCellRenderer.java new file mode 100644 index 0000000000..3ab729f5cc --- /dev/null +++ b/iped-app/src/main/java/iped/app/ui/controls/CheckboxListCellRenderer.java @@ -0,0 +1,64 @@ +package iped.app.ui.controls; + +import java.awt.BorderLayout; +import java.awt.Color; +import java.awt.Component; +import java.awt.event.ActionListener; +import java.util.function.Predicate; + +import javax.swing.JCheckBox; +import javax.swing.JList; +import javax.swing.JPanel; +import javax.swing.ListCellRenderer; + +public class CheckboxListCellRenderer extends JPanel implements ListCellRenderer { + JCheckBox enabledCheckBox = new JCheckBox(); + Predicate isEnabled; + int maxStringWidth = 0; + boolean indexedPredicate = false; + + public CheckboxListCellRenderer(Predicate isEnabled) { + this.setLayout(new BorderLayout()); + this.isEnabled = isEnabled; + this.setBackground(Color.white); + add(enabledCheckBox, BorderLayout.CENTER); + } + + public CheckboxListCellRenderer(Predicate isEnabled, boolean indexedPredicate) { + this(isEnabled); + this.indexedPredicate = indexedPredicate; + } + + @Override + public Component getListCellRendererComponent(JList list, E value, int index, boolean isSelected, + boolean cellHasFocus) { + if (indexedPredicate) { + enabledCheckBox.setSelected(isEnabled.test(index)); + } else { + enabledCheckBox.setSelected(isEnabled.test(value)); + } + enabledCheckBox.setText(value.toString()); + + this.add(enabledCheckBox, BorderLayout.CENTER); + if (isSelected) { + this.setBackground(Color.BLUE); + enabledCheckBox.setForeground(Color.white); + } else { + this.setBackground(Color.white); + enabledCheckBox.setForeground(Color.black); + } + + return this; + } + + public int getMaxStringWidth() { + if (maxStringWidth < enabledCheckBox.getWidth()) { + maxStringWidth = enabledCheckBox.getWidth(); + } + return maxStringWidth; + } + + public void addCheckBoxActionListener(ActionListener l) { + enabledCheckBox.addActionListener(l); + } +} \ No newline at end of file From 16a5caf84fd9b66cc30c325db39c4942261c6fe8 Mon Sep 17 00:00:00 2001 From: "patrick.pdb" Date: Thu, 22 Feb 2024 09:49:12 -0400 Subject: [PATCH 02/10] '#1864 Adds support for multiple group selection. --- .../app/timelinegraph/IpedChartsPanel.java | 48 +++---- .../app/timelinegraph/TimeEventGroup.java | 15 ++- .../cache/IndexTimeStampCache.java | 16 ++- .../timelinegraph/cache/TimeIndexedMap.java | 3 - .../timelinegraph/cache/TimeStampCache.java | 2 + .../cache/persistance/CachePersistance.java | 23 +++- .../datasets/IpedTimelineDataset.java | 118 +++++++++++------- .../datasets/IpedTimelineDatasetManager.java | 25 ++-- 8 files changed, 159 insertions(+), 91 deletions(-) diff --git a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java index 3db9e45298..d38938c7da 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java @@ -8,10 +8,10 @@ import java.awt.GridLayout; import java.awt.Image; import java.awt.Rectangle; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; import java.awt.event.ComponentEvent; import java.awt.event.ComponentListener; -import java.awt.event.ItemEvent; -import java.awt.event.ItemListener; import java.awt.event.MouseAdapter; import java.awt.event.MouseEvent; import java.awt.font.TextAttribute; @@ -42,7 +42,6 @@ import javax.swing.DefaultListModel; import javax.swing.ImageIcon; -import javax.swing.JComboBox; import javax.swing.JLabel; import javax.swing.JList; import javax.swing.JPanel; @@ -100,6 +99,7 @@ import iped.app.ui.ClearFilterListener; import iped.app.ui.ColumnsManager; import iped.app.ui.controls.CheckboxListCellRenderer; +import iped.app.ui.controls.JComboCheckBox; import iped.app.ui.themes.ThemeManager; import iped.data.IItemId; import iped.engine.search.QueryBuilder; @@ -118,7 +118,7 @@ * @author Patrick Dalla Bernardina */ public class IpedChartsPanel extends JPanel implements ResultSetViewer, TableModelListener, ListSelectionListener, - IQueryFilterer, ClearFilterListener, ComponentListener, ItemListener { + IQueryFilterer, ClearFilterListener, ComponentListener, ActionListener { JTable resultsTable; IMultiSearchResultProvider resultsProvider; GUIProvider guiProvider; @@ -154,7 +154,7 @@ public class IpedChartsPanel extends JPanel implements ResultSetViewer, TableMod IpedChartPanel chartPanel = null; JList legendList = new JList(); JScrollPane listScroller = new JScrollPane(legendList); - JComboBox tegCombo = new JComboBox<>(); + JComboCheckBox tegCombo = new JComboCheckBox<>(); IpedStackedXYBarRenderer renderer = null; XYLineAndShapeRenderer highlightsRenderer = new XYLineAndShapeRenderer(); @@ -180,7 +180,7 @@ public class IpedChartsPanel extends JPanel implements ResultSetViewer, TableMod Color fgColor; Color bgColor; - private TimeEventGroup currentTeGroup = TimeEventGroup.BASIC_EVENTS; + private ArrayList selectedTeGroups = new ArrayList(); private static final String resPath = '/' + App.class.getPackageName().replace('.', '/') + '/'; @@ -205,6 +205,7 @@ public IpedChartsPanel(boolean b) { super(b); this.setLayout(new GridLayout()); + } class LegendCellRenderer extends JLabel implements ListCellRenderer { @@ -299,15 +300,17 @@ public void init(JTable resultsTable, IMultiSearchResultProvider resultsProvider chartPanel = new IpedChartPanel(chart, this); - tegCombo.setRenderer(new CheckboxListCellRenderer(new Predicate() { - @Override - public boolean test(TimeEventGroup t) { - return true; - } - })); - chartPanel.add(tegCombo); - tegCombo.addItemListener(this); + CheckboxListCellRenderer cblcRenderer = new CheckboxListCellRenderer( + new Predicate() { + @Override + public boolean test(TimeEventGroup t) { + return selectedTeGroups.contains(t); + } + }); + tegCombo.setRenderer(cblcRenderer); + chartPanel.add(tegCombo); + tegCombo.addActionListener(this); legendListModel = new DefaultListModel(); legendList.setModel(legendListModel); @@ -418,15 +421,15 @@ public HashMap createDataSets() { if (selectedBookmarks.size() > 0 && chartPanel.getSplitByBookmark()) { for (String bookmark : selectedBookmarks) { result.put(bookmark, - ipedTimelineDatasetManager.getBestDataset(timePeriodClass, currentTeGroup, bookmark)); + ipedTimelineDatasetManager.getBestDataset(timePeriodClass, selectedTeGroups, bookmark)); } } else if (selectedCategories.size() > 0 && chartPanel.getSplitByCategory()) { for (String category : selectedCategories) { result.put(category, - ipedTimelineDatasetManager.getBestDataset(timePeriodClass, currentTeGroup, category)); + ipedTimelineDatasetManager.getBestDataset(timePeriodClass, selectedTeGroups, category)); } } else { - result.put("Items", ipedTimelineDatasetManager.getBestDataset(timePeriodClass, currentTeGroup, null)); + result.put("Items", ipedTimelineDatasetManager.getBestDataset(timePeriodClass, selectedTeGroups, null)); } return result; } catch (Exception e) { @@ -1112,10 +1115,13 @@ public static String[] getOrdToEventName() { } @Override - public void itemStateChanged(ItemEvent e) { - if (e.getStateChange() == ItemEvent.SELECTED) { - currentTeGroup = (TimeEventGroup) e.getItem(); - refreshChart(true); + public void actionPerformed(ActionEvent e) { + TimeEventGroup teGroup = (TimeEventGroup) tegCombo.getSelectedItem(); + if (selectedTeGroups.contains(teGroup)) { + selectedTeGroups.remove(teGroup); + } else { + selectedTeGroups.add(teGroup); } + refreshChart(false); } } \ No newline at end of file diff --git a/iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java b/iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java index ee47b2c4a4..c433c9241b 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java @@ -1,7 +1,10 @@ package iped.app.timelinegraph; +import java.util.Collection; import java.util.HashSet; +import org.roaringbitmap.RoaringBitmap; + /** * Class that represents a collection of event to use as a filter to graph * events viewing. Also, the cache will be managed/persisted based on this event @@ -14,6 +17,7 @@ public class TimeEventGroup { public static final TimeEventGroup BASIC_EVENTS = new TimeEventGroup("BasicProperties"); HashSet eventNames = new HashSet(); + RoaringBitmap eventOrds = new RoaringBitmap(); String name; @@ -50,12 +54,21 @@ public String getName() { return name; } - public void addEvent(String eventName) { + public void addEvent(String eventName, int ord) { eventNames.add(eventName); + eventOrds.add(ord); } @Override public String toString() { return name; } + + public Collection getEventNames() { + return eventNames; + } + + public RoaringBitmap getEventOrds() { + return eventOrds; + } } \ No newline at end of file diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java index 11745982f7..af4984922d 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java @@ -37,7 +37,7 @@ public class IndexTimeStampCache implements TimeStampCache { IMultiSearchResultProvider resultsProvider; IpedChartsPanel ipedChartsPanel; TimeZone timezone; - TimeEventGroup teGroup = TimeEventGroup.ALL_EVENTS;//default TimeEventGroup + TimeEventGroup teGroup;// default TimeEventGroup TimeIndexedMap newCache = new TimeIndexedMap(); @@ -73,7 +73,7 @@ public void run() { for (Class periodClasses : periodClassesToCache) { CachePersistance cp = CachePersistance.getInstance(); try { - TimeIndexedMap c = cp.loadNewCache(periodClasses); + TimeIndexedMap c = cp.loadNewCache(teGroup, periodClasses); if (c != null) { cacheExists = true; } @@ -137,7 +137,7 @@ public void run() { } else { CachePersistance cp = CachePersistance.getInstance(); for (Class periodClasses : periodClassesToCache) { - newCache.setIndexFile(periodClasses.getSimpleName(), cp.getBaseDir()); + newCache.setIndexFile(teGroup, periodClasses.getSimpleName(), cp.getBaseDir()); } newCache.createOrLoadUpperPeriodIndex(this); } @@ -318,4 +318,14 @@ public TimeEventGroup getTimeEventGroup() { return this.teGroup; } + @Override + public boolean isFromEventGroup(ArrayList selectedTeGroups) { + for (TimeEventGroup teGroup : selectedTeGroups) { + if (isFromEventGroup(teGroup)) { + return true; + } + } + return false; + } + } \ No newline at end of file diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java index b89eb233cb..bfeb6fee26 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java @@ -75,9 +75,6 @@ public void setIndexFile(TimeEventGroup teGroup, String periodName, File f) thro } } - public void setIndexFile(String periodName, File f) throws IOException { - } - Date lastStartDate = null; Date lastEndDate = null; diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java index 01597bb767..fc2bd42c8f 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java @@ -29,4 +29,6 @@ public interface TimeStampCache extends Runnable { public void setTimeEventGroup(TimeEventGroup teGroup); public TimeEventGroup getTimeEventGroup(); + + public boolean isFromEventGroup(ArrayList selectedTeGroups); } diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java index a2129c1f93..b8705aaed6 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java @@ -136,14 +136,25 @@ public void run() { } } - public TimeIndexedMap loadNewCache(Class className) throws IOException { + public TimeIndexedMap loadNewCache(TimeEventGroup teGroup, Class className) + throws IOException { TimeIndexedMap newCache = null; - for (File f : baseDir.listFiles()) { - if (f.getName().equals(className.getSimpleName())) { - newCache = new TimeIndexedMap(); - newCache.setIndexFile(className.getSimpleName(), baseDir); - break; + File[] files = baseDir.listFiles(); + if (files != null) { + for (File f : files) { + if (f.getName().equals(teGroup.getName())) { + File[] files2 = f.listFiles(); + if (files2 != null) { + for (File f2 : files2) { + if (f2.getName().equals(className.getSimpleName())) { + newCache = new TimeIndexedMap(); + newCache.setIndexFile(teGroup, className.getSimpleName(), baseDir); + break; + } + } + } + } } } diff --git a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java index b5eaaab37d..f4b89c416c 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java @@ -1,8 +1,10 @@ package iped.app.timelinegraph.datasets; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Calendar; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -10,8 +12,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadPoolExecutor; @@ -45,8 +45,10 @@ import iped.app.timelinegraph.IpedChartPanel; import iped.app.timelinegraph.IpedChartsPanel; import iped.app.timelinegraph.IpedDateAxis; +import iped.app.timelinegraph.TimeEventGroup; import iped.app.timelinegraph.cache.CacheEventEntry; import iped.app.timelinegraph.cache.CacheTimePeriodEntry; +import iped.app.timelinegraph.cache.CombinedIterators; import iped.app.timelinegraph.cache.EventTimestampCache; import iped.app.timelinegraph.cache.TimeIndexedMap; import iped.app.timelinegraph.cache.TimeStampCache; @@ -95,8 +97,7 @@ public class IpedTimelineDataset extends AbstractIntervalXYDataset implements Cl SortedSetDocValues timeEventGroupValues; IpedChartsPanel ipedChartsPanel; - SortedSet eventTypes = new TreeSet(); - String[] eventTypesArray; + RoaringBitmap eventOrds = new RoaringBitmap(); LeafReader reader; static ThreadPoolExecutor queriesThreadPool = new ThreadPoolExecutor(5, 10, 20000, TimeUnit.MILLISECONDS, new LinkedBlockingQueue());// pool of concurrent group of itens in a dataset beeing populated @@ -222,6 +223,9 @@ public void run() { for (int i = 0; i < threadCtsEnd; i++) { CacheTimePeriodEntry ct = threadLocalCts[i]; for (CacheEventEntry ce : ct.getEvents()) { + if (!eventOrds.contains(ce.getEventOrd())) { + continue; + } if (ce.docIds != null) { Count count = new Count(); for (int docId : ce.docIds) { @@ -349,8 +353,7 @@ public void caseSearchFilterLoad() throws Exception { result = csf.get(); if (result.getLength() > 0) { - TimeStampCache cache = ipedChartsPanel.getIpedTimelineDatasetManager().getCache(); - TimeIndexedMap a = (TimeIndexedMap) cache.getNewCache(); + Collection caches = ipedChartsPanel.getIpedTimelineDatasetManager().getCaches(); String className = ipedChartsPanel.getTimePeriodClass().getSimpleName(); @@ -388,62 +391,83 @@ public void caseSearchFilterLoad() throws Exception { cacheWindowEndDate = new Date(ipedChartsPanel.getChartPanel().removeNextFromDatePart(endDate).getTime() - 1); } - try (RandomAccessBufferedFileInputStream sfis = a.getTmpCacheSfis(cache.getTimeEventGroup(), - className)) { - if (sfis != null) { - Iterator it = a.iterator(cache.getTimeEventGroup(), className, sfis, - startDate, endDate); - while (it != null && it.hasNext()) { - ipedChartsPanel.getIpedTimelineDatasetManager().waitMemory();// method to wait available mem to continue. - if (cancelled) { - break; - } + ArrayList toCloseAtEnd = new ArrayList(); + try { + Iterator[] iterators = new Iterator[caches.size()]; + int i = 0; + eventOrds.clear(); + for (TimeStampCache cache : caches) { + TimeIndexedMap cacheMap = (TimeIndexedMap) cache.getNewCache(); + TimeEventGroup teGroup = cache.getTimeEventGroup(); + RandomAccessBufferedFileInputStream sfis = cacheMap + .getTmpCacheSfis(teGroup, className); + if (sfis == null) { + throw new IOException("Temporary timeline cache not found for " + cache.getTimeEventGroup() + + " group:" + className); + } + eventOrds.or(teGroup.getEventOrds()); + toCloseAtEnd.add(sfis); + iterators[i] = cacheMap.iterator(teGroup, className, sfis, startDate, + endDate); + i++; + } + CombinedIterators it = new CombinedIterators(iterators); + while (it != null && it.hasNext()) { + ipedChartsPanel.getIpedTimelineDatasetManager().waitMemory();// method to wait available mem to continue. + if (cancelled) { + break; + } - CacheTimePeriodEntry ctpe = it.next(); + CacheTimePeriodEntry ctpe = it.next(); - boolean remove = false; - if (!fullrange) { - if (ctpe.getDate().before(startDate)) { - if (ctpe.getDate().getTime() > cacheWindowStartDate.getTime()) { - if (!memoryWindowCache.contains(ctpe)) { - beforecache.addFirst(ctpe); - memoryWindowCache.add(ctpe); - } - remove = false; - } else { - // remove from memoryCacheWindow - remove = true; - } - } else if (ctpe.getDate().after(endDate)) { - if (ctpe.getDate().getTime() < cacheWindowEndDate.getTime()) { - if (!memoryWindowCache.contains(ctpe)) { - aftercache.add(ctpe); - memoryWindowCache.add(ctpe); - } - remove = false; - } else { - // remove from memoryCacheWindow - remove = true; + + boolean remove = false; + + if (!fullrange) { + if (ctpe.getDate().before(startDate)) { + if (ctpe.getDate().getTime() > cacheWindowStartDate.getTime()) { + if (!memoryWindowCache.contains(ctpe)) { + beforecache.addFirst(ctpe); + memoryWindowCache.add(ctpe); } - } else {// inside visible window + remove = false; + } else { + // remove from memoryCacheWindow + remove = true; + } + } else if (ctpe.getDate().after(endDate)) { + if (ctpe.getDate().getTime() < cacheWindowEndDate.getTime()) { if (!memoryWindowCache.contains(ctpe)) { - visibleIntervalCache.add(ctpe); + aftercache.add(ctpe); memoryWindowCache.add(ctpe); } remove = false; + } else { + // remove from memoryCacheWindow + remove = true; } - } else { + } else {// inside visible window if (!memoryWindowCache.contains(ctpe)) { visibleIntervalCache.add(ctpe); memoryWindowCache.add(ctpe); } + remove = false; } - if (remove) { - TimePeriod t = ipedChartsPanel.getDomainAxis().getDateOnConfiguredTimePeriod(ipedChartsPanel.getTimePeriodClass(), ctpe.getDate()); - accumulator.remove(t); - memoryWindowCache.remove(ctpe); + } else { + if (!memoryWindowCache.contains(ctpe)) { + visibleIntervalCache.add(ctpe); + memoryWindowCache.add(ctpe); } } + if (remove) { + TimePeriod t = ipedChartsPanel.getDomainAxis().getDateOnConfiguredTimePeriod(ipedChartsPanel.getTimePeriodClass(), ctpe.getDate()); + accumulator.remove(t); + memoryWindowCache.remove(ctpe); + } + } + } finally { + for (RandomAccessBufferedFileInputStream sfis : toCloseAtEnd) { + sfis.close(); } } diff --git a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java index b2fe9351c4..c208976381 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java @@ -41,7 +41,7 @@ public class IpedTimelineDatasetManager { List timeStampCaches = new ArrayList<>(); volatile boolean isCacheLoaded = false; - TimeStampCache selectedTimeStampCache; + ArrayList selectedTimeStampCaches = new ArrayList(); public IpedTimelineDatasetManager(IpedChartsPanel ipedChartsPanel) { this.ipedChartsPanel = ipedChartsPanel; @@ -71,18 +71,20 @@ public IpedTimelineDatasetManager(IpedChartsPanel ipedChartsPanel) { } - public AbstractIntervalXYDataset getBestDataset(Class timePeriodClass, TimeEventGroup teGroup, + public AbstractIntervalXYDataset getBestDataset(Class timePeriodClass, + ArrayList selectedTeGroups, String splitValue) { + selectedTimeStampCaches.clear(); try { for (TimeStampCache timeStampCache : timeStampCaches) { if (timeStampCache.hasTimePeriodClassToCache(timePeriodClass) - && timeStampCache.isFromEventGroup(teGroup)) { - selectedTimeStampCache = timeStampCache; - IpedTimelineDataset result = new IpedTimelineDataset(this, ipedChartsPanel.getResultsProvider(), splitValue); - return result; + && timeStampCache.isFromEventGroup(selectedTeGroups)) { + selectedTimeStampCaches.add(timeStampCache); } } - return null; + IpedTimelineDataset result = new IpedTimelineDataset(this, ipedChartsPanel.getResultsProvider(), + splitValue); + return result; } catch (Exception e) { e.printStackTrace(); } @@ -128,8 +130,8 @@ public void startCacheCreation() { } } - public TimeStampCache getCache() { - return selectedTimeStampCache; + public Collection getCaches() { + return selectedTimeStampCaches; } public IpedChartsPanel getIpedChartsPanel() { @@ -171,6 +173,7 @@ public Collection getTimeEventGroupsFromMetadataPrefix() { groupNames = new HashMap(); String[] cachedEventNames = ipedChartsPanel.getOrdToEventName(); + int ord = 0; for (String eventName : cachedEventNames) { String groupName; TimeEventGroup teGroup = TimeEventGroup.BASIC_EVENTS; @@ -188,9 +191,11 @@ public Collection getTimeEventGroupsFromMetadataPrefix() { } - teGroup.addEvent(eventName); + teGroup.addEvent(eventName, ord); + ord++; } } return groupNames.values(); } + } From a241b6abf186a35fbf3dea56304c5423b46eebda Mon Sep 17 00:00:00 2001 From: "patrick.pdb" Date: Thu, 22 Feb 2024 10:15:35 -0400 Subject: [PATCH 03/10] '#1864 Removes reference to unused class --- .../src/main/java/iped/app/timelinegraph/IpedChartsPanel.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java index d38938c7da..cdf768bd8f 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java @@ -42,6 +42,7 @@ import javax.swing.DefaultListModel; import javax.swing.ImageIcon; +import javax.swing.JComboBox; import javax.swing.JLabel; import javax.swing.JList; import javax.swing.JPanel; @@ -99,7 +100,6 @@ import iped.app.ui.ClearFilterListener; import iped.app.ui.ColumnsManager; import iped.app.ui.controls.CheckboxListCellRenderer; -import iped.app.ui.controls.JComboCheckBox; import iped.app.ui.themes.ThemeManager; import iped.data.IItemId; import iped.engine.search.QueryBuilder; @@ -154,7 +154,7 @@ public class IpedChartsPanel extends JPanel implements ResultSetViewer, TableMod IpedChartPanel chartPanel = null; JList legendList = new JList(); JScrollPane listScroller = new JScrollPane(legendList); - JComboCheckBox tegCombo = new JComboCheckBox<>(); + JComboBox tegCombo = new JComboBox<>(); IpedStackedXYBarRenderer renderer = null; XYLineAndShapeRenderer highlightsRenderer = new XYLineAndShapeRenderer(); From 1bd7230f75868226b81e72c9ed34a7be84ec29d9 Mon Sep 17 00:00:00 2001 From: "patrick.pdb" Date: Thu, 22 Feb 2024 10:31:33 -0400 Subject: [PATCH 04/10] '#1864 Changes base dir name for timecache persistence to avoid conflicts with old indexes. --- .../timelinegraph/cache/persistance/CachePersistance.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java index b8705aaed6..a382952fe1 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java @@ -62,6 +62,8 @@ public class CachePersistance { static CachePersistance singleton = new CachePersistance(); + static final String TIMECACHE_BASE_FOLDER = "timecachegroups"; + public static CachePersistance getInstance() { return singleton; } @@ -78,7 +80,7 @@ public CachePersistance() { startDir = new File(App.get().casesPathFile.getParentFile(), "iped-multicases"); } - startDir = new File(startDir, "timecache"); + startDir = new File(startDir, TIMECACHE_BASE_FOLDER); startDir.mkdirs(); bitstreamSerializeFile = new File(startDir, "bitstreamSerialize"); @@ -101,7 +103,7 @@ public void getTempBaseDir(File startDir) { baseDir = new File(startDir, uuid); if (!baseDir.exists() && !baseDir.mkdirs()) { // cache doesn't exist and folder is not writable, use user.home for caches - startDir = new File(System.getProperty("user.home"), ".iped/timecache"); + startDir = new File(System.getProperty("user.home"), ".iped/" + TIMECACHE_BASE_FOLDER); baseDir = new File(startDir, uuid); baseDir.mkdirs(); } From 1a4c7459978071b30b76d234337704a3c8c4e7b2 Mon Sep 17 00:00:00 2001 From: "patrick.pdb" Date: Thu, 22 Feb 2024 10:33:27 -0400 Subject: [PATCH 05/10] '#1864 Corrects class name (english) --- .../app/timelinegraph/cache/IndexTimeStampCache.java | 8 ++++---- .../app/timelinegraph/cache/PersistedArrayList.java | 8 ++++---- .../iped/app/timelinegraph/cache/TimeIndexedMap.java | 6 +++--- .../{CachePersistance.java => CachePersistence.java} | 10 +++++----- .../timelinegraph/datasets/IpedTimelineDataset.java | 4 ++-- 5 files changed, 18 insertions(+), 18 deletions(-) rename iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/{CachePersistance.java => CachePersistence.java} (98%) diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java index af4984922d..1a21a76da5 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java @@ -22,7 +22,7 @@ import iped.app.timelinegraph.IpedChartsPanel; import iped.app.timelinegraph.TimeEventGroup; -import iped.app.timelinegraph.cache.persistance.CachePersistance; +import iped.app.timelinegraph.cache.persistance.CachePersistence; import iped.engine.core.Manager; import iped.viewers.api.IMultiSearchResultProvider; @@ -71,7 +71,7 @@ public void run() { periodClassesToCache.add(ipedChartsPanel.getTimePeriodClass()); } for (Class periodClasses : periodClassesToCache) { - CachePersistance cp = CachePersistance.getInstance(); + CachePersistence cp = CachePersistence.getInstance(); try { TimeIndexedMap c = cp.loadNewCache(teGroup, periodClasses); if (c != null) { @@ -111,7 +111,7 @@ public void run() { if (Manager.getInstance() != null && Manager.getInstance().isProcessingFinished()) { } - CachePersistance cp = CachePersistance.getInstance(); + CachePersistence cp = CachePersistence.getInstance(); cp.saveNewCache(this); cacheLoaders.clear(); @@ -135,7 +135,7 @@ public void run() { } } else { - CachePersistance cp = CachePersistance.getInstance(); + CachePersistence cp = CachePersistence.getInstance(); for (Class periodClasses : periodClassesToCache) { newCache.setIndexFile(teGroup, periodClasses.getSimpleName(), cp.getBaseDir()); } diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/PersistedArrayList.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/PersistedArrayList.java index 4563e89393..9a3de99c9f 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/PersistedArrayList.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/PersistedArrayList.java @@ -16,8 +16,8 @@ import org.apache.logging.log4j.Logger; import org.jfree.data.time.TimePeriod; -import iped.app.timelinegraph.cache.persistance.CachePersistance; -import iped.app.timelinegraph.cache.persistance.CachePersistance.CacheFileIterator; +import iped.app.timelinegraph.cache.persistance.CachePersistence; +import iped.app.timelinegraph.cache.persistance.CachePersistence.CacheFileIterator; import iped.app.timelinegraph.datasets.IpedTimelineDatasetManager; public class PersistedArrayList implements Set { @@ -176,7 +176,7 @@ private void flush() { final int lflushCount = flushCount++; - CachePersistance cp = CachePersistance.getInstance(); + CachePersistence cp = CachePersistence.getInstance(); Future f = cp.cachePersistanceExecutor.submit(new Runnable() { @Override public void run() { @@ -201,7 +201,7 @@ public void run() { } public void removeFlushes() { - CachePersistance.getInstance().cachePersistanceExecutor.execute(new Runnable() { + CachePersistence.getInstance().cachePersistanceExecutor.execute(new Runnable() { @Override public void run() { for (File f : flushFiles) { diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java index bfeb6fee26..786577d59e 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeIndexedMap.java @@ -15,7 +15,7 @@ import org.apache.pdfbox.io.RandomAccessBufferedFileInputStream; import iped.app.timelinegraph.TimeEventGroup; -import iped.app.timelinegraph.cache.persistance.CachePersistance; +import iped.app.timelinegraph.cache.persistance.CachePersistence; /** * @author Patrick Dalla Bernardina @@ -171,7 +171,7 @@ class ResultIterator implements Iterator { private String className; boolean useCache = false; private TimelineCache timelineCache; - CachePersistance cp = CachePersistance.getInstance(); + CachePersistence cp = CachePersistence.getInstance(); int countRead = 0;// counters to log number of cache reads int countCache = 0;// counters to log number of disk reads private Reference[] lsoftcache;// soft reference cache (a more complete cache but with SoftReferences) @@ -271,7 +271,7 @@ public CacheTimePeriodEntry next() { }; public void createOrLoadUpperPeriodIndex(IndexTimeStampCache indexTimeStampCache) { - CachePersistance cp = CachePersistance.getInstance(); + CachePersistence cp = CachePersistence.getInstance(); if (upperPeriodIndex.size() == 0) { TimelineCache timelineCache = TimelineCache.get(indexTimeStampCache.getTimeEventGroup()); for (Iterator iterator = indexTimeStampCache.getPeriodClassesToCache().iterator(); iterator.hasNext();) { diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistence.java similarity index 98% rename from iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java rename to iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistence.java index a382952fe1..fcad00a1c0 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistance.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/persistance/CachePersistence.java @@ -44,10 +44,10 @@ import iped.utils.IOUtil; /* - * Class implementing method for timeline chart cache persistance + * Class implementing method for timeline chart cache persistence */ -public class CachePersistance { +public class CachePersistence { File baseDir; HashMap pathsToCheck = new HashMap(); @@ -60,17 +60,17 @@ public class CachePersistance { static private boolean bitstreamSerializeAsDefault = true; - static CachePersistance singleton = new CachePersistance(); + static CachePersistence singleton = new CachePersistence(); static final String TIMECACHE_BASE_FOLDER = "timecachegroups"; - public static CachePersistance getInstance() { + public static CachePersistence getInstance() { return singleton; } static public ExecutorService cachePersistanceExecutor = Executors.newFixedThreadPool(1); - public CachePersistance() { + public CachePersistence() { File startDir; if (App.get().appCase.getAtomicSources().size() == 1) { // single case diff --git a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java index f4b89c416c..063941da22 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDataset.java @@ -52,7 +52,7 @@ import iped.app.timelinegraph.cache.EventTimestampCache; import iped.app.timelinegraph.cache.TimeIndexedMap; import iped.app.timelinegraph.cache.TimeStampCache; -import iped.app.timelinegraph.cache.persistance.CachePersistance; +import iped.app.timelinegraph.cache.persistance.CachePersistence; import iped.app.ui.App; import iped.app.ui.CaseSearcherFilter; import iped.app.ui.Messages; @@ -1197,7 +1197,7 @@ public void run() { t.start(); } - CachePersistance cp = new CachePersistance(); + CachePersistence cp = new CachePersistence(); public void addValue(Count count, TimePeriod t, String eventType) { if (min == null || t.getStart().before(min.getStart())) { From 49264000e12a7c5e9f66268dfabdc55e7f7d2874 Mon Sep 17 00:00:00 2001 From: "patrick.pdb" Date: Thu, 22 Feb 2024 11:26:49 -0400 Subject: [PATCH 06/10] '#1864 better position and render of time event group selection combo box. --- .../app/timelinegraph/IpedChartsPanel.java | 13 +++++++--- .../ui/controls/CheckboxListCellRenderer.java | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java index cdf768bd8f..0122d839b1 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java @@ -1,5 +1,6 @@ package iped.app.timelinegraph; +import java.awt.BorderLayout; import java.awt.Color; import java.awt.Component; import java.awt.Dimension; @@ -154,6 +155,7 @@ public class IpedChartsPanel extends JPanel implements ResultSetViewer, TableMod IpedChartPanel chartPanel = null; JList legendList = new JList(); JScrollPane listScroller = new JScrollPane(legendList); + JPanel legendPane = new JPanel(); JComboBox tegCombo = new JComboBox<>(); IpedStackedXYBarRenderer renderer = null; @@ -309,8 +311,8 @@ public boolean test(TimeEventGroup t) { } }); tegCombo.setRenderer(cblcRenderer); - chartPanel.add(tegCombo); tegCombo.addActionListener(this); + tegCombo.setMaximumSize(new Dimension(100, 0)); legendListModel = new DefaultListModel(); legendList.setModel(legendListModel); @@ -362,7 +364,12 @@ public void check(MouseEvent e) { ttm.setEnabled(true); splitPane.setTopComponent(chartPanel); - splitPane.setBottomComponent(listScroller); + + legendPane.setPreferredSize(new Dimension(Integer.MAX_VALUE, 80)); + legendPane.setLayout(new BorderLayout()); + legendPane.add(tegCombo, BorderLayout.NORTH); + legendPane.add(listScroller, BorderLayout.CENTER); + splitPane.setBottomComponent(legendPane); splitPane.setVisible(false); chartPanel.setPopupMenu(null); @@ -544,7 +551,7 @@ public void run() { self.remove(splitPane); self.add(splitPane); splitPane.setTopComponent(chartPanel); - splitPane.setBottomComponent(listScroller); + splitPane.setBottomComponent(legendPane); splitPane.setVisible(true); // hide hidden events diff --git a/iped-app/src/main/java/iped/app/ui/controls/CheckboxListCellRenderer.java b/iped-app/src/main/java/iped/app/ui/controls/CheckboxListCellRenderer.java index 3ab729f5cc..8a46441cf7 100644 --- a/iped-app/src/main/java/iped/app/ui/controls/CheckboxListCellRenderer.java +++ b/iped-app/src/main/java/iped/app/ui/controls/CheckboxListCellRenderer.java @@ -7,12 +7,14 @@ import java.util.function.Predicate; import javax.swing.JCheckBox; +import javax.swing.JLabel; import javax.swing.JList; import javax.swing.JPanel; import javax.swing.ListCellRenderer; public class CheckboxListCellRenderer extends JPanel implements ListCellRenderer { JCheckBox enabledCheckBox = new JCheckBox(); + JLabel selectedValues = new JLabel(); Predicate isEnabled; int maxStringWidth = 0; boolean indexedPredicate = false; @@ -32,6 +34,30 @@ public CheckboxListCellRenderer(Predicate isEnabled, boolean indexedPredicate) { @Override public Component getListCellRendererComponent(JList list, E value, int index, boolean isSelected, boolean cellHasFocus) { + if (index == -1) { + + StringBuffer sb = new StringBuffer(); + sb.append(" "); + for (int i = 0; i < list.getModel().getSize(); i++) { + Object lvalue = list.getModel().getElementAt(i); + boolean include = false; + if (indexedPredicate) { + if (isEnabled.test(i)) { + include = true; + } + } else { + if (isEnabled.test(lvalue)) { + include = true; + } + } + if (include) { + sb.append(lvalue); + sb.append(","); + } + } + selectedValues.setText(sb.toString()); + return selectedValues; + } if (indexedPredicate) { enabledCheckBox.setSelected(isEnabled.test(index)); } else { From 4dcadac24cd748ae3fdcb768b9575d595866079f Mon Sep 17 00:00:00 2001 From: "patrick.pdb" Date: Thu, 22 Feb 2024 14:40:52 -0400 Subject: [PATCH 07/10] '#1864 Implements on demmand cache creation/loading, i. e., if group was never selected its cache is not created/loaded. --- .../app/timelinegraph/IpedChartsPanel.java | 15 ++---- .../cache/IndexTimeStampCache.java | 15 +++--- .../timelinegraph/cache/TimeStampCache.java | 2 + .../datasets/IpedTimelineDatasetManager.java | 53 +++++++++---------- 4 files changed, 39 insertions(+), 46 deletions(-) diff --git a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java index 0122d839b1..e7230e69f3 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java @@ -193,6 +193,8 @@ public IpedChartsPanel() { } combinedPlot.setDomainPannable(true); + selectedTeGroups.add(TimeEventGroup.BASIC_EVENTS); + toolTipGenerator = new XYToolTipGenerator() { @Override public String generateToolTip(XYDataset dataset, int series, int item) { @@ -311,7 +313,6 @@ public boolean test(TimeEventGroup t) { } }); tegCombo.setRenderer(cblcRenderer); - tegCombo.addActionListener(this); tegCombo.setMaximumSize(new Dimension(100, 0)); legendListModel = new DefaultListModel(); @@ -618,15 +619,6 @@ public void setDockableContainer(DefaultSingleCDockable dockable) { public void changed(CDockableLocationEvent dockableEvent) { if (!isUpdated && dockableEvent.isShowingChanged()) { refreshChart(); - if (!loadingCacheStarted.getAndSet(true)) { - Runnable r = new Runnable() { - @Override - public void run() { - ipedTimelineDatasetManager.startCacheCreation(); - } - }; - new Thread(r).start(); - } } } }; @@ -1102,11 +1094,14 @@ public void notifyCaseDataChanged() { populateEventNames.run(); this.ipedTimelineDatasetManager = new IpedTimelineDatasetManager(this); + // updates tegCombo with updated time event groups of the case + tegCombo.removeActionListener(this); tegCombo.removeAllItems(); tegCombo.addItem(TimeEventGroup.BASIC_EVENTS); for (TimeEventGroup teGroup : ipedTimelineDatasetManager.getTimeEventGroupsFromMetadataPrefix()) { tegCombo.addItem(teGroup); } + tegCombo.addActionListener(this); this.dataSetUpdated.set(false); } diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java index 1a21a76da5..5257e46756 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java @@ -13,6 +13,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.logging.log4j.LogManager; @@ -40,6 +41,7 @@ public class IndexTimeStampCache implements TimeStampCache { TimeEventGroup teGroup;// default TimeEventGroup TimeIndexedMap newCache = new TimeIndexedMap(); + AtomicBoolean loading = new AtomicBoolean(false); public IndexTimeStampCache(IpedChartsPanel ipedChartsPanel, IMultiSearchResultProvider resultsProvider) { this.resultsProvider = resultsProvider; @@ -57,6 +59,7 @@ public IndexTimeStampCache(IpedChartsPanel ipedChartsPanel, IMultiSearchResultPr @Override public void run() { + loading.set(true); int oldPriority = Thread.currentThread().getPriority(); Thread.currentThread().setPriority(Thread.MIN_PRIORITY); try { @@ -70,6 +73,7 @@ public void run() { if (periodClassesToCache.size() == 0) { periodClassesToCache.add(ipedChartsPanel.getTimePeriodClass()); } + for (Class periodClasses : periodClassesToCache) { CachePersistence cp = CachePersistence.getInstance(); try { @@ -155,13 +159,6 @@ public void addTimePeriodClassToCache(Class timePeriodClas } public boolean hasTimePeriodClassToCache(Class timePeriodClass) { - try { - timeStampCacheSemaphore.acquire();// pause until cache is populated - } catch (InterruptedException e) { - e.printStackTrace(); - } finally { - timeStampCacheSemaphore.release(); - } return periodClassesToCache.contains(timePeriodClass); } @@ -328,4 +325,8 @@ public boolean isFromEventGroup(ArrayList selectedTeGroups) { return false; } + public boolean isLoading() { + return loading.get(); + } + } \ No newline at end of file diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java index fc2bd42c8f..a3cb18f16c 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/TimeStampCache.java @@ -31,4 +31,6 @@ public interface TimeStampCache extends Runnable { public TimeEventGroup getTimeEventGroup(); public boolean isFromEventGroup(ArrayList selectedTeGroups); + + public boolean isLoading(); } diff --git a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java index c208976381..1824ff4ca1 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/datasets/IpedTimelineDatasetManager.java @@ -9,7 +9,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -76,12 +75,18 @@ public AbstractIntervalXYDataset getBestDataset(Class time String splitValue) { selectedTimeStampCaches.clear(); try { + for (TimeEventGroup tGroup : selectedTeGroups) { + createCaches(tGroup);// assures caches of the group are created/loaded + } + for (TimeStampCache timeStampCache : timeStampCaches) { - if (timeStampCache.hasTimePeriodClassToCache(timePeriodClass) - && timeStampCache.isFromEventGroup(selectedTeGroups)) { - selectedTimeStampCaches.add(timeStampCache); + if (timeStampCache.isFromEventGroup(selectedTeGroups)) { + if (timeStampCache.hasTimePeriodClassToCache(timePeriodClass)) { + selectedTimeStampCaches.add(timeStampCache); + } } } + IpedTimelineDataset result = new IpedTimelineDataset(this, ipedChartsPanel.getResultsProvider(), splitValue); return result; @@ -91,43 +96,33 @@ public AbstractIntervalXYDataset getBestDataset(Class time return null; } - /* - * Start the creation of cache for timeline chart - */ - public void startCacheCreation() { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - + public void createCaches(TimeEventGroup teGroup) { int poolSize = 1; + int totalItems = ipedChartsPanel.getResultsProvider().getIPEDSource().getTotalItems(); if (getAvailableMemory() > totalItems * 100) { poolSize = (int) Math.ceil((float) Runtime.getRuntime().availableProcessors() / 2f); - } else { - logger.info("Only {}MB of free memory for {} total items. Timeline index creation will occur sequentially. ", Runtime.getRuntime().freeMemory(), totalItems); } + ExecutorService threadPool = Executors.newFixedThreadPool(poolSize); boolean first = true; for (TimeStampCache timeStampCache : timeStampCaches) { - Future future = threadPool.submit(timeStampCache); - // first loads the Day cache alone to speed up it, then run others in parallel - if (first) { - first = false; - try { - future.get(); - } catch (InterruptedException | ExecutionException e) { - e.printStackTrace(); + if (!timeStampCache.isLoading()) { + if (timeStampCache.isFromEventGroup(teGroup)) { + Future future = threadPool.submit(timeStampCache); + // first loads the Day cache alone to speed up it, then run others in parallel + if (first) { + first = false; + try { + future.get(); + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + } } } } threadPool.shutdown(); - try { - threadPool.awaitTermination(12, TimeUnit.HOURS); - } catch (InterruptedException e) { - e.printStackTrace(); - } } public Collection getCaches() { From 3c822e9a4555a4f0a7142c81da6be9602bec97a8 Mon Sep 17 00:00:00 2001 From: "patrick.pdb" Date: Thu, 22 Feb 2024 15:05:48 -0400 Subject: [PATCH 08/10] '#1864 improves progress information. --- .../app/timelinegraph/IpedChartsPanel.java | 29 ++++++++++++++----- .../cache/IndexTimeStampCache.java | 5 ++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java index e7230e69f3..89a23d6801 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java @@ -41,6 +41,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Predicate; +import javax.swing.BoxLayout; import javax.swing.DefaultListModel; import javax.swing.ImageIcon; import javax.swing.JComboBox; @@ -164,7 +165,9 @@ public class IpedChartsPanel extends JPanel implements ResultSetViewer, TableMod String metadataToBreakChart = null; ImageIcon loading = null; - JLabel loadingLabel; + JPanel loadingPanel = new JPanel(); + JLabel loadingLabelImage; + JLabel loadingLabelText; IpedSplitPane splitPane; @@ -377,8 +380,15 @@ public void check(MouseEvent e) { this.addComponentListener(this); loading = (ImageIcon) new ImageIcon(IconUtil.class.getResource(resPath + "loading.gif")); - loadingLabel = new JLabel("", loading, JLabel.CENTER); - this.add(loadingLabel); + loadingLabelImage = new JLabel("", loading, JLabel.CENTER); + loadingLabelText = new JLabel("", JLabel.CENTER); + loadingPanel.setAlignmentY(Component.CENTER_ALIGNMENT); + loadingPanel.setLayout(new BoxLayout(loadingPanel, BoxLayout.Y_AXIS)); + loadingLabelImage.setAlignmentX(Component.CENTER_ALIGNMENT); + loadingLabelText.setAlignmentX(Component.CENTER_ALIGNMENT); + loadingPanel.add(loadingLabelImage, BorderLayout.CENTER); + loadingPanel.add(loadingLabelText, BorderLayout.CENTER); + this.add(loadingPanel); domainAxis.setTickMarkPosition(DateTickMarkPosition.START); domainAxis.setLowerMargin(0.01); @@ -420,6 +430,7 @@ public HashMap createAlertDataSets() { } public HashMap createDataSets() { + HashMap result = new HashMap(); try { Set selectedBookmarks = guiProvider.getSelectedBookmarks(); @@ -482,8 +493,8 @@ public Future refreshChart(boolean resetDomainRange) { IpedChartsPanel self = this; self.remove(splitPane); - self.remove(loadingLabel); - self.add(loadingLabel); + self.remove(loadingPanel); + self.add(loadingPanel); self.repaint(); if (swRefresh != null) { @@ -548,7 +559,7 @@ public void run() { } if (chart != null && !isCancelled()) { - self.remove(loadingLabel); + self.remove(loadingPanel); self.remove(splitPane); self.add(splitPane); splitPane.setTopComponent(chartPanel); @@ -1060,7 +1071,7 @@ public void setLegendList(JList legendList) { @Override public void componentResized(ComponentEvent e) { - this.remove(loadingLabel); + this.remove(loadingPanel); this.remove(splitPane); this.add(splitPane); } @@ -1126,4 +1137,8 @@ public void actionPerformed(ActionEvent e) { } refreshChart(false); } + + public void info(String info) { + loadingLabelText.setText(info); + } } \ No newline at end of file diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java index 5257e46756..be492b9ff2 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java @@ -89,6 +89,8 @@ public void run() { if (!cacheExists) { Date d1 = new Date(); logger.info("Starting to build time cache of [{}]...", periodClassesToCache.toString()); + ipedChartsPanel.info("Starting to build time cache of [" + periodClassesToCache.toString() + "] to " + + teGroup.getName() + " event type group"); ArrayList cacheLoaders = new ArrayList(); @@ -139,6 +141,9 @@ public void run() { } } else { + ipedChartsPanel.info("Starting to load time cache of [" + periodClassesToCache.toString() + "] to " + + teGroup.getName() + " event type group"); + CachePersistence cp = CachePersistence.getInstance(); for (Class periodClasses : periodClassesToCache) { newCache.setIndexFile(teGroup, periodClasses.getSimpleName(), cp.getBaseDir()); From 2f032806b60834a4b26c3855c4e385dcaff29fe0 Mon Sep 17 00:00:00 2001 From: "patrick.pdb" Date: Thu, 22 Feb 2024 15:22:23 -0400 Subject: [PATCH 09/10] '#1864 If no group is selected, "BasicProperties" is selected automatically. --- .../src/main/java/iped/app/timelinegraph/IpedChartsPanel.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java index 89a23d6801..a0f68c7386 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java @@ -1135,6 +1135,9 @@ public void actionPerformed(ActionEvent e) { } else { selectedTeGroups.add(teGroup); } + if (selectedTeGroups.isEmpty()) { + selectedTeGroups.add(TimeEventGroup.BASIC_EVENTS); + } refreshChart(false); } From 9cc8dd1ea1c74aa5cc116571a678e291652cb0bb Mon Sep 17 00:00:00 2001 From: "patrick.pdb" Date: Thu, 22 Feb 2024 15:41:30 -0400 Subject: [PATCH 10/10] '#1864 Some Internationalization --- .../localization/iped-desktop-messages.properties | 5 ++++- .../localization/iped-desktop-messages_de_DE.properties | 3 +++ .../localization/iped-desktop-messages_es_AR.properties | 3 +++ .../localization/iped-desktop-messages_it_IT.properties | 3 +++ .../localization/iped-desktop-messages_pt_BR.properties | 3 +++ .../java/iped/app/timelinegraph/IpedChartsPanel.java | 4 ++-- .../main/java/iped/app/timelinegraph/TimeEventGroup.java | 5 ++++- .../app/timelinegraph/cache/IndexTimeStampCache.java | 9 +++++---- 8 files changed, 27 insertions(+), 8 deletions(-) diff --git a/iped-app/resources/localization/iped-desktop-messages.properties b/iped-app/resources/localization/iped-desktop-messages.properties index 5993438261..b9c5806d3f 100644 --- a/iped-app/resources/localization/iped-desktop-messages.properties +++ b/iped-app/resources/localization/iped-desktop-messages.properties @@ -394,4 +394,7 @@ TimeLineGraph.uncheckEventItems=Uncheck corresponding event type items IntervalDefinitionDialog.Title=Time line chart defined ranges IntervalDefinitionDialog.dateFormat=MM/dd/yyyy IntervalDefinitionDialog.hourFormat=HH:mm:ss -IntervalDefinitionDialog.Exit=Exit \ No newline at end of file +IntervalDefinitionDialog.Exit=Exit +TimeEventGroup.BasicProperties=Basic Properties +IndexTimeStampCache.loadingMessage=Starting to load time cache of [{}] to {} event type group. +IndexTimeStampCache.buildingMessage=Starting to build time cache of [{}] to {} event type group. \ No newline at end of file diff --git a/iped-app/resources/localization/iped-desktop-messages_de_DE.properties b/iped-app/resources/localization/iped-desktop-messages_de_DE.properties index c236c38c55..912198c39f 100644 --- a/iped-app/resources/localization/iped-desktop-messages_de_DE.properties +++ b/iped-app/resources/localization/iped-desktop-messages_de_DE.properties @@ -395,3 +395,6 @@ IntervalDefinitionDialog.Title=Zeitlininechart festgelegte Bereiche IntervalDefinitionDialog.dateFormat=MM/dd/yyyy IntervalDefinitionDialog.hourFormat=HH:mm:ss IntervalDefinitionDialog.Exit=Beenden +TimeEventGroup.BasicProperties=Basic Properties(TBT) +IndexTimeStampCache.loadingMessage=Starting to load time cache of [{}] to {} event type group.(TBT) +IndexTimeStampCache.buildingMessage=Starting to build time cache of [{}] to {} event type group.(TBT) \ No newline at end of file diff --git a/iped-app/resources/localization/iped-desktop-messages_es_AR.properties b/iped-app/resources/localization/iped-desktop-messages_es_AR.properties index f93f17b617..d9f940165e 100644 --- a/iped-app/resources/localization/iped-desktop-messages_es_AR.properties +++ b/iped-app/resources/localization/iped-desktop-messages_es_AR.properties @@ -395,3 +395,6 @@ IntervalDefinitionDialog.Title=Rangos definidos en el gráfico de tiempo IntervalDefinitionDialog.dateFormat=MM/dd/yyyy IntervalDefinitionDialog.hourFormat=HH:mm:ss IntervalDefinitionDialog.Exit=Salir +TimeEventGroup.BasicProperties=Basic Properties(TBT) +IndexTimeStampCache.loadingMessage=Starting to load time cache of [{}] to {} event type group.(TBT) +IndexTimeStampCache.buildingMessage=Starting to build time cache of [{}] to {} event type group.(TBT) \ No newline at end of file diff --git a/iped-app/resources/localization/iped-desktop-messages_it_IT.properties b/iped-app/resources/localization/iped-desktop-messages_it_IT.properties index a82204ca1c..f5f31ceeb0 100644 --- a/iped-app/resources/localization/iped-desktop-messages_it_IT.properties +++ b/iped-app/resources/localization/iped-desktop-messages_it_IT.properties @@ -395,3 +395,6 @@ IntervalDefinitionDialog.Title=Grafico sequenza temporale con intervalli definit IntervalDefinitionDialog.dateFormat=dd/MM/yyyy IntervalDefinitionDialog.hourFormat=HH:mm:ss IntervalDefinitionDialog.Exit=Esci +TimeEventGroup.BasicProperties=Basic Properties(TBT) +IndexTimeStampCache.loadingMessage=Starting to load time cache of [{}] to {} event type group.(TBT) +IndexTimeStampCache.buildingMessage=Starting to build time cache of [{}] to {} event type group.(TBT) \ No newline at end of file diff --git a/iped-app/resources/localization/iped-desktop-messages_pt_BR.properties b/iped-app/resources/localization/iped-desktop-messages_pt_BR.properties index ab36198ed1..b7fa22c6ac 100644 --- a/iped-app/resources/localization/iped-desktop-messages_pt_BR.properties +++ b/iped-app/resources/localization/iped-desktop-messages_pt_BR.properties @@ -395,3 +395,6 @@ IntervalDefinitionDialog.Title=Intervalos definidos no gráfico IntervalDefinitionDialog.dateFormat=MM/dd/yyyy IntervalDefinitionDialog.hourFormat=HH:mm:ss IntervalDefinitionDialog.Exit=Sair +TimeEventGroup.BasicProperties=Propriedades básicas +IndexTimeStampCache.loadingMessage=Carregando índice de [{}] para o grupo de eventos {}. +IndexTimeStampCache.buildingMessage=Construindo índice de [{}] para o grupo de eventos {}. \ No newline at end of file diff --git a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java index a0f68c7386..75213c27a8 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/IpedChartsPanel.java @@ -1141,7 +1141,7 @@ public void actionPerformed(ActionEvent e) { refreshChart(false); } - public void info(String info) { - loadingLabelText.setText(info); + public void info(String info, String period, String groupname) { + loadingLabelText.setText(info.replaceFirst("\\{\\}", period).replaceFirst("\\{\\}", groupname)); } } \ No newline at end of file diff --git a/iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java b/iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java index c433c9241b..7143a0cc8e 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/TimeEventGroup.java @@ -5,6 +5,8 @@ import org.roaringbitmap.RoaringBitmap; +import iped.app.ui.Messages; + /** * Class that represents a collection of event to use as a filter to graph * events viewing. Also, the cache will be managed/persisted based on this event @@ -14,7 +16,8 @@ */ public class TimeEventGroup { public static final TimeEventGroup ALL_EVENTS = new TimeEventGroup(); - public static final TimeEventGroup BASIC_EVENTS = new TimeEventGroup("BasicProperties"); + public static final TimeEventGroup BASIC_EVENTS = new TimeEventGroup( + Messages.get("TimeEventGroup.BasicProperties")); HashSet eventNames = new HashSet(); RoaringBitmap eventOrds = new RoaringBitmap(); diff --git a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java index be492b9ff2..c4b5d49f7c 100644 --- a/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java +++ b/iped-app/src/main/java/iped/app/timelinegraph/cache/IndexTimeStampCache.java @@ -24,6 +24,7 @@ import iped.app.timelinegraph.IpedChartsPanel; import iped.app.timelinegraph.TimeEventGroup; import iped.app.timelinegraph.cache.persistance.CachePersistence; +import iped.app.ui.Messages; import iped.engine.core.Manager; import iped.viewers.api.IMultiSearchResultProvider; @@ -89,8 +90,8 @@ public void run() { if (!cacheExists) { Date d1 = new Date(); logger.info("Starting to build time cache of [{}]...", periodClassesToCache.toString()); - ipedChartsPanel.info("Starting to build time cache of [" + periodClassesToCache.toString() + "] to " - + teGroup.getName() + " event type group"); + ipedChartsPanel.info(Messages.get("IndexTimeStampCache.buildingMessage"), + periodClassesToCache.toString(), teGroup.toString()); ArrayList cacheLoaders = new ArrayList(); @@ -141,8 +142,8 @@ public void run() { } } else { - ipedChartsPanel.info("Starting to load time cache of [" + periodClassesToCache.toString() + "] to " - + teGroup.getName() + " event type group"); + ipedChartsPanel.info(Messages.get("IndexTimeStampCache.loadingMessage"), + periodClassesToCache.toString(), teGroup.toString()); CachePersistence cp = CachePersistence.getInstance(); for (Class periodClasses : periodClassesToCache) {