Changed on-disk format

xivsupport/src/main/java/gg/xp/xivsupport/events/misc/RawEventStorage.java

@@ -5,6 +5,7 @@
 import gg.xp.reevent.scan.HandleEvents;
 import gg.xp.reevent.scan.LiveOnly;
 import gg.xp.xivsupport.events.debug.DebugCommand;
+import gg.xp.xivsupport.persistence.Compressible;
 import gg.xp.xivsupport.persistence.PersistenceProvider;
 import gg.xp.xivsupport.persistence.settings.BooleanSetting;
 import gg.xp.xivsupport.persistence.settings.IntSetting;
@@ -79,9 +80,7 @@ public void storeEvent(EventContext context, Event event) {
 	@LiveOnly
 	@HandleEvents(order = Integer.MAX_VALUE)
 	public void writeEventToDisk(EventContext context, Event event) {
-		if (event.shouldSave() && saveToDisk.get()) {
-			exs.submit(() -> this.saveEventToDisk(event));
-		}
+		exs.submit(() -> this.saveEventToDisk(event));
 	}
 
 	@HandleEvents
@@ -102,25 +101,30 @@ public IntSetting getMaxEventsStoredSetting() {
 	}
 
 	private void saveEventToDisk(Event event) {
-		try {
-			if (eventSaveStream == null) {
-				String userDataDir = System.getenv("APPDATA");
-				Path sessionsDir = Paths.get(userDataDir, "triggevent", "sessions", sessionName);
-				File sessionsDirFile = sessionsDir.toFile();
-				sessionsDirFile.mkdirs();
-				if (!sessionsDirFile.exists() && sessionsDirFile.isDirectory()) {
-					log.error("Error saving to disk! Could not make dirs: {}", sessionsDirFile.getAbsolutePath());
-					return;
+		if (event instanceof Compressible) {
+			((Compressible) event).compress();
+		}
+		if (event.shouldSave() && saveToDisk.get()) {
+			try {
+				if (eventSaveStream == null) {
+					String userDataDir = System.getenv("APPDATA");
+					Path sessionsDir = Paths.get(userDataDir, "triggevent", "sessions", sessionName);
+					File sessionsDirFile = sessionsDir.toFile();
+					sessionsDirFile.mkdirs();
+					if (!sessionsDirFile.exists() && sessionsDirFile.isDirectory()) {
+						log.error("Error saving to disk! Could not make dirs: {}", sessionsDirFile.getAbsolutePath());
+						return;
+					}
+					File file = Paths.get(sessionsDir.toString(), "session.oos.gz").toFile();
+					FileOutputStream fileOutputStream = new FileOutputStream(file, true);
+					GZIPOutputStream compressedOutputStream = new GZIPOutputStream(fileOutputStream);
+					eventSaveStream = new ObjectOutputStream(compressedOutputStream);
 				}
-				File file = Paths.get(sessionsDir.toString(), "session.oos.gz").toFile();
-				FileOutputStream fileOutputStream = new FileOutputStream(file, true);
-				GZIPOutputStream compressedOutputStream = new GZIPOutputStream(fileOutputStream);
-				eventSaveStream = new ObjectOutputStream(compressedOutputStream);
+				eventSaveStream.writeObject(event);
+			}
+			catch (IOException e) {
+				log.error("Error saving to disk!", e);
 			}
-			eventSaveStream.writeObject(event);
-		}
-		catch (IOException e) {
-			log.error("Error saving to disk!", e);
 		}
 	}
 

xivsupport/src/main/java/gg/xp/xivsupport/events/ws/ActWsRawMsg.java

@@ -3,12 +3,13 @@
 import gg.xp.reevent.events.BaseEvent;
 import gg.xp.reevent.events.SystemEvent;
 import gg.xp.xivsupport.events.misc.Compressor;
+import gg.xp.xivsupport.persistence.Compressible;
 import org.intellij.lang.annotations.Language;
 
 import java.time.Instant;
 
 @SystemEvent
-public class ActWsRawMsg extends BaseEvent {
+public class ActWsRawMsg extends BaseEvent implements Compressible {
 	private static final long serialVersionUID = -7390177233308577948L;
 	private String rawMsgData;
 	private byte[] compressed;
@@ -28,13 +29,23 @@ public String getRawMsgData() {
 	}
 
 	@Override
-	public void setPumpFinishedAt(Instant pumpedAt) {
-		super.setPumpFinishedAt(pumpedAt);
+	public void compress() {
 		byte[] compressed = Compressor.compressStringToBytes(rawMsgData);
 		// Only compress if it actually saves memory
 		if (compressed.length < rawMsgData.length()) {
 			this.compressed = compressed;
+			// TODO: this isn't working right - getting NPEs in getRawMsgData when deserializing.
+			// Guessing that there's a concurrency issue where the thread doing the serialization
+			// is seeing both fields as null.
 			rawMsgData = null;
 		}
 	}
+
+	@Override
+	public void decompress() {
+		String raw = this.rawMsgData;
+		if (raw == null) {
+			this.rawMsgData = Compressor.uncompressBytesToString(compressed);
+		}
+	}
 }

xivsupport/src/main/java/gg/xp/xivsupport/eventstorage/EventReader.java

@@ -8,13 +8,15 @@
 import java.io.ObjectInputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.zip.GZIPInputStream;
 
 public class EventReader {
 
 	public static List<Event> readEventsFromResource(String resourcePath) {
 		InputStream stream = EventReader.class.getResourceAsStream(resourcePath);
 		List<Event> events;
-		try (ObjectInputStream ois = new ObjectInputStream(stream)) {
+		try (GZIPInputStream gzip = new GZIPInputStream(stream);
+		     ObjectInputStream ois = new ObjectInputStream(gzip)) {
 			// TODO: security
 			ObjectInputFilter filter = filterInfo -> {
 				if (filterInfo.serialClass() == null) {

xivsupport/src/main/java/gg/xp/xivsupport/gui/GuiMain.java

@@ -202,7 +202,7 @@ private class SystemTabPanel extends JPanel {
 			c.weightx = 0;
 			c.gridwidth = 1;
 			XivStateStatus xivStateStatus = new XivStateStatus();
-			xivStateStatus.setMinimumSize(new Dimension(200, 200));
+			xivStateStatus.setMinimumSize(new Dimension(250, 200));
 			xivStateStatus.setPreferredSize(xivStateStatus.getMinimumSize());
 //			xivStateStatus.setPreferredSize(new Dimension(100, 250));
 			add(xivStateStatus, c);
@@ -283,7 +283,7 @@ public XivStateStatus() {
 
 			KeyValueDisplaySet leftItems = new KeyValueDisplaySet(List.of(
 					new KeyValuePairDisplay<>(
-							"Player Name",
+							"Name",
 							new JLabel(),
 							() -> {
 								XivPlayerCharacter player = state.get(XivState.class).getPlayer();
@@ -292,7 +292,7 @@ public XivStateStatus() {
 							JLabel::setText
 					),
 					new KeyValuePairDisplay<>(
-							"Zone Name",
+							"Zone",
 							new JLabel(),
 							() -> {
 								XivZone zone = state.get(XivState.class).getZone();
@@ -301,7 +301,7 @@ public XivStateStatus() {
 							JLabel::setText
 					),
 					new KeyValuePairDisplay<>(
-							"Player Job",
+							"Job",
 							new JLabel(),
 							() -> {
 								XivPlayerCharacter player = state.get(XivState.class).getPlayer();
@@ -310,7 +310,7 @@ public XivStateStatus() {
 							JLabel::setText
 					),
 					new KeyValuePairDisplay<>(
-							"Player Level",
+							"Level",
 							new JLabel(),
 							() -> {
 								XivPlayerCharacter player = state.get(XivState.class).getPlayer();
@@ -319,7 +319,7 @@ public XivStateStatus() {
 							JLabel::setText
 					),
 					new KeyValuePairDisplay<>(
-							"Player World",
+							"World",
 							new JLabel(),
 							() -> {
 								XivPlayerCharacter player = state.get(XivState.class).getPlayer();

xivsupport/src/main/java/gg/xp/xivsupport/persistence/Compressible.java

@@ -0,0 +1,6 @@
+package gg.xp.xivsupport.persistence;
+
+public interface Compressible {
+	void compress();
+	void decompress();
+}

xivsupport/src/test/java/gg/xp/xivsupport/eventstorage/EventSerializationTests.java

@@ -24,4 +24,10 @@ public void readBigSampleFile() {
 		Assert.assertEquals(events.size(), 644);
 	}
 
+	@Test
+	public void readCompressedSampleFile() {
+		List<Event> events = EventReader.readEventsFromResource("/testsession3.oos.gz");
+		Assert.assertEquals(events.size(), 25790);
+	}
+
 }

xivsupport/src/test/java/gg/xp/xivsupport/gui/GuiWithImportedData.java

@@ -2,17 +2,22 @@
 
 import com.formdev.flatlaf.FlatDarculaLaf;
 import gg.xp.reevent.events.AutoEventDistributor;
+import gg.xp.reevent.events.BasicEventQueue;
 import gg.xp.reevent.events.InitEvent;
 import gg.xp.reevent.events.Event;
 import gg.xp.reevent.events.EventMaster;
+import gg.xp.xivsupport.events.misc.RawEventStorage;
 import gg.xp.xivsupport.eventstorage.EventReader;
 import gg.xp.xivsupport.sys.XivMain;
 import org.picocontainer.MutablePicoContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.swing.*;
 import java.util.List;
 
 public class GuiWithImportedData {
+	private static final Logger log = LoggerFactory.getLogger(GuiWithImportedData.class);
 	public static void main(String[] args) throws InterruptedException {
 		try {
 			UIManager.setLookAndFeel(new FlatDarculaLaf());
@@ -21,19 +26,36 @@ public static void main(String[] args) throws InterruptedException {
 			throw new RuntimeException(t);
 		}
 		MutablePicoContainer pico = XivMain.testingMasterInit();
-		EventMaster master = pico.getComponent(EventMaster.class);
-		pico.addComponent(GuiMain.class);
 		AutoEventDistributor dist = pico.getComponent(AutoEventDistributor.class);
 		dist.acceptEvent(new InitEvent());
+		RawEventStorage raw = pico.getComponent(RawEventStorage.class);
+		raw.getMaxEventsStoredSetting().set(1_000_000);
+		BasicEventQueue queue = pico.getComponent(BasicEventQueue.class);
+		pico.addComponent(GuiMain.class);
 		pico.getComponent(GuiMain.class);
 
-		List<Event> events = EventReader.readEventsFromResource("/testsession2.oos");
-		for (Event event : events) {
-			Thread.sleep(100);
-//			master.pushEvent(event);
-			dist.acceptEvent(event);
-		}
+		long start = System.currentTimeMillis();
+		List<Event> events = EventReader.readEventsFromResource("/testsession5.oos.gz");
+		long read = System.currentTimeMillis();
+		events.stream()
+//				.limit(10000)
+				.peek((e) -> doSleep(1))
+				.forEach(dist::acceptEvent);
+		queue.waitDrain();
+		long finish = System.currentTimeMillis();
+		log.info("Imported Event Count: {}", events.size());
+		log.info("Processed Event Count: {}", raw.getEvents().size());
+		log.info("Time to read, decompress, and deserialize: {}ms", read - start);
+		log.info("Time to replay: {}ms", finish - read);
+
 
+	}
+	private static void doSleep(long ms) {
+		try {
+			Thread.sleep(ms);
+		}
+		catch (InterruptedException e) {
 
+		}
 	}
 }