From 928e84692356c4a31d17eaa6dd7f3ffd6bec5bf6 Mon Sep 17 00:00:00 2001 From: xp Date: Wed, 1 Dec 2021 16:21:35 -0800 Subject: [PATCH] Changed on-disk format --- .../events/misc/RawEventStorage.java | 44 ++++++++++--------- .../xp/xivsupport/events/ws/ActWsRawMsg.java | 17 +++++-- .../xivsupport/eventstorage/EventReader.java | 4 +- .../java/gg/xp/xivsupport/gui/GuiMain.java | 12 ++--- .../xivsupport/persistence/Compressible.java | 6 +++ .../eventstorage/EventSerializationTests.java | 6 +++ .../xivsupport/gui/GuiWithImportedData.java | 38 ++++++++++++---- 7 files changed, 89 insertions(+), 38 deletions(-) create mode 100644 xivsupport/src/main/java/gg/xp/xivsupport/persistence/Compressible.java diff --git a/xivsupport/src/main/java/gg/xp/xivsupport/events/misc/RawEventStorage.java b/xivsupport/src/main/java/gg/xp/xivsupport/events/misc/RawEventStorage.java index 213a0de6eaef..a7fb4950f701 100644 --- a/xivsupport/src/main/java/gg/xp/xivsupport/events/misc/RawEventStorage.java +++ b/xivsupport/src/main/java/gg/xp/xivsupport/events/misc/RawEventStorage.java @@ -5,6 +5,7 @@ import gg.xp.reevent.scan.HandleEvents; import gg.xp.reevent.scan.LiveOnly; import gg.xp.xivsupport.events.debug.DebugCommand; +import gg.xp.xivsupport.persistence.Compressible; import gg.xp.xivsupport.persistence.PersistenceProvider; import gg.xp.xivsupport.persistence.settings.BooleanSetting; import gg.xp.xivsupport.persistence.settings.IntSetting; @@ -79,9 +80,7 @@ public void storeEvent(EventContext context, Event event) { @LiveOnly @HandleEvents(order = Integer.MAX_VALUE) public void writeEventToDisk(EventContext context, Event event) { - if (event.shouldSave() && saveToDisk.get()) { - exs.submit(() -> this.saveEventToDisk(event)); - } + exs.submit(() -> this.saveEventToDisk(event)); } @HandleEvents @@ -102,25 +101,30 @@ public IntSetting getMaxEventsStoredSetting() { } private void saveEventToDisk(Event event) { - try { - if (eventSaveStream == null) { - String userDataDir = System.getenv("APPDATA"); - Path sessionsDir = Paths.get(userDataDir, "triggevent", "sessions", sessionName); - File sessionsDirFile = sessionsDir.toFile(); - sessionsDirFile.mkdirs(); - if (!sessionsDirFile.exists() && sessionsDirFile.isDirectory()) { - log.error("Error saving to disk! Could not make dirs: {}", sessionsDirFile.getAbsolutePath()); - return; + if (event instanceof Compressible) { + ((Compressible) event).compress(); + } + if (event.shouldSave() && saveToDisk.get()) { + try { + if (eventSaveStream == null) { + String userDataDir = System.getenv("APPDATA"); + Path sessionsDir = Paths.get(userDataDir, "triggevent", "sessions", sessionName); + File sessionsDirFile = sessionsDir.toFile(); + sessionsDirFile.mkdirs(); + if (!sessionsDirFile.exists() && sessionsDirFile.isDirectory()) { + log.error("Error saving to disk! Could not make dirs: {}", sessionsDirFile.getAbsolutePath()); + return; + } + File file = Paths.get(sessionsDir.toString(), "session.oos.gz").toFile(); + FileOutputStream fileOutputStream = new FileOutputStream(file, true); + GZIPOutputStream compressedOutputStream = new GZIPOutputStream(fileOutputStream); + eventSaveStream = new ObjectOutputStream(compressedOutputStream); } - File file = Paths.get(sessionsDir.toString(), "session.oos.gz").toFile(); - FileOutputStream fileOutputStream = new FileOutputStream(file, true); - GZIPOutputStream compressedOutputStream = new GZIPOutputStream(fileOutputStream); - eventSaveStream = new ObjectOutputStream(compressedOutputStream); + eventSaveStream.writeObject(event); + } + catch (IOException e) { + log.error("Error saving to disk!", e); } - eventSaveStream.writeObject(event); - } - catch (IOException e) { - log.error("Error saving to disk!", e); } } diff --git a/xivsupport/src/main/java/gg/xp/xivsupport/events/ws/ActWsRawMsg.java b/xivsupport/src/main/java/gg/xp/xivsupport/events/ws/ActWsRawMsg.java index 906d7e7a7cca..05b04e359c96 100644 --- a/xivsupport/src/main/java/gg/xp/xivsupport/events/ws/ActWsRawMsg.java +++ b/xivsupport/src/main/java/gg/xp/xivsupport/events/ws/ActWsRawMsg.java @@ -3,12 +3,13 @@ import gg.xp.reevent.events.BaseEvent; import gg.xp.reevent.events.SystemEvent; import gg.xp.xivsupport.events.misc.Compressor; +import gg.xp.xivsupport.persistence.Compressible; import org.intellij.lang.annotations.Language; import java.time.Instant; @SystemEvent -public class ActWsRawMsg extends BaseEvent { +public class ActWsRawMsg extends BaseEvent implements Compressible { private static final long serialVersionUID = -7390177233308577948L; private String rawMsgData; private byte[] compressed; @@ -28,13 +29,23 @@ public String getRawMsgData() { } @Override - public void setPumpFinishedAt(Instant pumpedAt) { - super.setPumpFinishedAt(pumpedAt); + public void compress() { byte[] compressed = Compressor.compressStringToBytes(rawMsgData); // Only compress if it actually saves memory if (compressed.length < rawMsgData.length()) { this.compressed = compressed; + // TODO: this isn't working right - getting NPEs in getRawMsgData when deserializing. + // Guessing that there's a concurrency issue where the thread doing the serialization + // is seeing both fields as null. rawMsgData = null; } } + + @Override + public void decompress() { + String raw = this.rawMsgData; + if (raw == null) { + this.rawMsgData = Compressor.uncompressBytesToString(compressed); + } + } } diff --git a/xivsupport/src/main/java/gg/xp/xivsupport/eventstorage/EventReader.java b/xivsupport/src/main/java/gg/xp/xivsupport/eventstorage/EventReader.java index 7dea7499cc63..1a04bec6c152 100644 --- a/xivsupport/src/main/java/gg/xp/xivsupport/eventstorage/EventReader.java +++ b/xivsupport/src/main/java/gg/xp/xivsupport/eventstorage/EventReader.java @@ -8,13 +8,15 @@ import java.io.ObjectInputStream; import java.util.ArrayList; import java.util.List; +import java.util.zip.GZIPInputStream; public class EventReader { public static List readEventsFromResource(String resourcePath) { InputStream stream = EventReader.class.getResourceAsStream(resourcePath); List events; - try (ObjectInputStream ois = new ObjectInputStream(stream)) { + try (GZIPInputStream gzip = new GZIPInputStream(stream); + ObjectInputStream ois = new ObjectInputStream(gzip)) { // TODO: security ObjectInputFilter filter = filterInfo -> { if (filterInfo.serialClass() == null) { diff --git a/xivsupport/src/main/java/gg/xp/xivsupport/gui/GuiMain.java b/xivsupport/src/main/java/gg/xp/xivsupport/gui/GuiMain.java index 0af1c973371e..b90f54046ee5 100644 --- a/xivsupport/src/main/java/gg/xp/xivsupport/gui/GuiMain.java +++ b/xivsupport/src/main/java/gg/xp/xivsupport/gui/GuiMain.java @@ -202,7 +202,7 @@ private class SystemTabPanel extends JPanel { c.weightx = 0; c.gridwidth = 1; XivStateStatus xivStateStatus = new XivStateStatus(); - xivStateStatus.setMinimumSize(new Dimension(200, 200)); + xivStateStatus.setMinimumSize(new Dimension(250, 200)); xivStateStatus.setPreferredSize(xivStateStatus.getMinimumSize()); // xivStateStatus.setPreferredSize(new Dimension(100, 250)); add(xivStateStatus, c); @@ -283,7 +283,7 @@ public XivStateStatus() { KeyValueDisplaySet leftItems = new KeyValueDisplaySet(List.of( new KeyValuePairDisplay<>( - "Player Name", + "Name", new JLabel(), () -> { XivPlayerCharacter player = state.get(XivState.class).getPlayer(); @@ -292,7 +292,7 @@ public XivStateStatus() { JLabel::setText ), new KeyValuePairDisplay<>( - "Zone Name", + "Zone", new JLabel(), () -> { XivZone zone = state.get(XivState.class).getZone(); @@ -301,7 +301,7 @@ public XivStateStatus() { JLabel::setText ), new KeyValuePairDisplay<>( - "Player Job", + "Job", new JLabel(), () -> { XivPlayerCharacter player = state.get(XivState.class).getPlayer(); @@ -310,7 +310,7 @@ public XivStateStatus() { JLabel::setText ), new KeyValuePairDisplay<>( - "Player Level", + "Level", new JLabel(), () -> { XivPlayerCharacter player = state.get(XivState.class).getPlayer(); @@ -319,7 +319,7 @@ public XivStateStatus() { JLabel::setText ), new KeyValuePairDisplay<>( - "Player World", + "World", new JLabel(), () -> { XivPlayerCharacter player = state.get(XivState.class).getPlayer(); diff --git a/xivsupport/src/main/java/gg/xp/xivsupport/persistence/Compressible.java b/xivsupport/src/main/java/gg/xp/xivsupport/persistence/Compressible.java new file mode 100644 index 000000000000..4f0d0b568da6 --- /dev/null +++ b/xivsupport/src/main/java/gg/xp/xivsupport/persistence/Compressible.java @@ -0,0 +1,6 @@ +package gg.xp.xivsupport.persistence; + +public interface Compressible { + void compress(); + void decompress(); +} diff --git a/xivsupport/src/test/java/gg/xp/xivsupport/eventstorage/EventSerializationTests.java b/xivsupport/src/test/java/gg/xp/xivsupport/eventstorage/EventSerializationTests.java index fa77ff401e2b..b16c8d90172c 100644 --- a/xivsupport/src/test/java/gg/xp/xivsupport/eventstorage/EventSerializationTests.java +++ b/xivsupport/src/test/java/gg/xp/xivsupport/eventstorage/EventSerializationTests.java @@ -24,4 +24,10 @@ public void readBigSampleFile() { Assert.assertEquals(events.size(), 644); } + @Test + public void readCompressedSampleFile() { + List events = EventReader.readEventsFromResource("/testsession3.oos.gz"); + Assert.assertEquals(events.size(), 25790); + } + } diff --git a/xivsupport/src/test/java/gg/xp/xivsupport/gui/GuiWithImportedData.java b/xivsupport/src/test/java/gg/xp/xivsupport/gui/GuiWithImportedData.java index ff5c69e2ca68..17b7636bb59d 100644 --- a/xivsupport/src/test/java/gg/xp/xivsupport/gui/GuiWithImportedData.java +++ b/xivsupport/src/test/java/gg/xp/xivsupport/gui/GuiWithImportedData.java @@ -2,17 +2,22 @@ import com.formdev.flatlaf.FlatDarculaLaf; import gg.xp.reevent.events.AutoEventDistributor; +import gg.xp.reevent.events.BasicEventQueue; import gg.xp.reevent.events.InitEvent; import gg.xp.reevent.events.Event; import gg.xp.reevent.events.EventMaster; +import gg.xp.xivsupport.events.misc.RawEventStorage; import gg.xp.xivsupport.eventstorage.EventReader; import gg.xp.xivsupport.sys.XivMain; import org.picocontainer.MutablePicoContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.swing.*; import java.util.List; public class GuiWithImportedData { + private static final Logger log = LoggerFactory.getLogger(GuiWithImportedData.class); public static void main(String[] args) throws InterruptedException { try { UIManager.setLookAndFeel(new FlatDarculaLaf()); @@ -21,19 +26,36 @@ public static void main(String[] args) throws InterruptedException { throw new RuntimeException(t); } MutablePicoContainer pico = XivMain.testingMasterInit(); - EventMaster master = pico.getComponent(EventMaster.class); - pico.addComponent(GuiMain.class); AutoEventDistributor dist = pico.getComponent(AutoEventDistributor.class); dist.acceptEvent(new InitEvent()); + RawEventStorage raw = pico.getComponent(RawEventStorage.class); + raw.getMaxEventsStoredSetting().set(1_000_000); + BasicEventQueue queue = pico.getComponent(BasicEventQueue.class); + pico.addComponent(GuiMain.class); pico.getComponent(GuiMain.class); - List events = EventReader.readEventsFromResource("/testsession2.oos"); - for (Event event : events) { - Thread.sleep(100); -// master.pushEvent(event); - dist.acceptEvent(event); - } + long start = System.currentTimeMillis(); + List events = EventReader.readEventsFromResource("/testsession5.oos.gz"); + long read = System.currentTimeMillis(); + events.stream() +// .limit(10000) + .peek((e) -> doSleep(1)) + .forEach(dist::acceptEvent); + queue.waitDrain(); + long finish = System.currentTimeMillis(); + log.info("Imported Event Count: {}", events.size()); + log.info("Processed Event Count: {}", raw.getEvents().size()); + log.info("Time to read, decompress, and deserialize: {}ms", read - start); + log.info("Time to replay: {}ms", finish - read); + + } + private static void doSleep(long ms) { + try { + Thread.sleep(ms); + } + catch (InterruptedException e) { + } } }