Cleanup WorldEdit-CLI data version systems (#2347)

* Start cleanup of WorldEdit-CLI data versions

* Use a record for the data file

* Use cassette-deck to download data files rather than bundling

Sketchy implementation atm, better will follow - this "works" as-is without any special handling of missing versions. Intention is to get the "next closest version" when it's not available

* Fix up build errors

* Review changes

* Fix a potential race condition

verification/build.gradle.kts

@@ -43,7 +43,7 @@ tasks.check {
 // Generic setup for all tasks
 // Pull the version before our current version.
 val baseVersion = "(,${rootProject.version.toString().substringBefore("-SNAPSHOT")}["
-for (projectFragment in listOf("bukkit", "cli", "core", "fabric", "forge", "sponge")) {
+for (projectFragment in listOf("bukkit", "core", "fabric", "forge", "sponge")) {
     val capitalizedFragment =
         projectFragment.replaceFirstChar { if (it.isLowerCase()) it.titlecase(Locale.ROOT) else it.toString() }
     val proj = project(":worldedit-$projectFragment")

worldedit-cli/src/main/java/com/sk89q/worldedit/cli/CLIBlockCategoryRegistry.java

@@ -30,7 +30,7 @@ public class CLIBlockCategoryRegistry implements BlockCategoryRegistry {
 
     @Override
     public Set<BlockType> getCategorisedByName(String category) {
-        return CLIWorldEdit.inst.getFileRegistries().getDataFile().blocktags.getOrDefault(category, Collections.emptyList()).stream()
+        return CLIWorldEdit.inst.getFileRegistries().getDataFile().blockTags().getOrDefault(category, Collections.emptyList()).stream()
                 .map(BlockType.REGISTRY::get)
                 .collect(Collectors.toSet());
     }

worldedit-cli/src/main/java/com/sk89q/worldedit/cli/CLIBlockRegistry.java

@@ -21,7 +21,7 @@
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
-import com.sk89q.worldedit.cli.data.FileRegistries;
+import com.sk89q.worldedit.cli.data.DataFile;
 import com.sk89q.worldedit.registry.state.BooleanProperty;
 import com.sk89q.worldedit.registry.state.DirectionalProperty;
 import com.sk89q.worldedit.registry.state.EnumProperty;
@@ -51,10 +51,10 @@ private Property<?> createProperty(String type, String key, List<String> values)
     @Nullable
     @Override
     public Map<String, ? extends Property<?>> getProperties(BlockType blockType) {
-        Map<String, FileRegistries.BlockProperty> properties =
-                CLIWorldEdit.inst.getFileRegistries().getDataFile().blocks.get(blockType.id()).properties;
-        Maps.EntryTransformer<String, FileRegistries.BlockProperty, Property<?>> entryTransform =
-            (key, value) -> createProperty(value.type, key, value.values);
+        Map<String, DataFile.BlockProperty> properties =
+                CLIWorldEdit.inst.getFileRegistries().getDataFile().blocks().get(blockType.id()).properties();
+        Maps.EntryTransformer<String, DataFile.BlockProperty, Property<?>> entryTransform =
+            (key, value) -> createProperty(value.type(), key, value.values());
         return ImmutableMap.copyOf(Maps.transformEntries(properties, entryTransform));
     }
 }

worldedit-cli/src/main/java/com/sk89q/worldedit/cli/CLICommandSender.java

@@ -43,14 +43,11 @@ public class CLICommandSender implements Actor {
      */
     private static final UUID DEFAULT_ID = UUID.fromString("a233eb4b-4cab-42cd-9fd9-7e7b9a3f74be");
 
-    private final CLIWorldEdit app;
     private final Logger sender;
 
-    public CLICommandSender(CLIWorldEdit app, Logger sender) {
-        checkNotNull(app);
+    public CLICommandSender(Logger sender) {
         checkNotNull(sender);
 
-        this.app = app;
         this.sender = sender;
     }
 

worldedit-cli/src/main/java/com/sk89q/worldedit/cli/CLIConfiguration.java

@@ -29,10 +29,6 @@ public CLIConfiguration(CLIWorldEdit app) {
         super(app.getWorkingDir().resolve("worldedit.properties"));
     }
 
-    @Override
-    protected void loadExtra() {
-    }
-
     @Override
     public Path getWorkingDirectoryPath() {
         return CLIWorldEdit.inst.getWorkingDir();

worldedit-cli/src/main/java/com/sk89q/worldedit/cli/CLIItemCategoryRegistry.java

@@ -29,7 +29,7 @@ public class CLIItemCategoryRegistry implements ItemCategoryRegistry {
 
     @Override
     public Set<ItemType> getCategorisedByName(String category) {
-        return CLIWorldEdit.inst.getFileRegistries().getDataFile().itemtags.get(category).stream()
+        return CLIWorldEdit.inst.getFileRegistries().getDataFile().itemTags().get(category).stream()
                 .map(ItemType.REGISTRY::get)
                 .collect(Collectors.toSet());
     }

worldedit-cli/src/main/java/com/sk89q/worldedit/cli/CLIPlatform.java

@@ -25,7 +25,6 @@
 import com.sk89q.worldedit.extension.platform.Capability;
 import com.sk89q.worldedit.extension.platform.Preference;
 import com.sk89q.worldedit.util.SideEffect;
-import com.sk89q.worldedit.world.DataFixer;
 import com.sk89q.worldedit.world.World;
 import com.sk89q.worldedit.world.entity.EntityTypes;
 import com.sk89q.worldedit.world.registry.Registries;
@@ -67,11 +66,6 @@ public void setDataVersion(int dataVersion) {
         this.dataVersion = dataVersion;
     }
 
-    @Override
-    public DataFixer getDataFixer() {
-        return null;
-    }
-
     @Override
     public boolean isValidMobType(String type) {
         return EntityTypes.get(type) != null;
@@ -132,6 +126,9 @@ public CLIConfiguration getConfiguration() {
 
     @Override
     public String getVersion() {
+        if (app.getInternalVersion() == null) {
+            return "unknown"; // Run from IDE
+        }
         return app.getInternalVersion();
     }
 
@@ -142,7 +139,7 @@ public String getPlatformName() {
 
     @Override
     public String getPlatformVersion() {
-        return app.getInternalVersion();
+        return this.getVersion();
     }
 
     @Override

worldedit-cli/src/main/java/com/sk89q/worldedit/cli/CLIWorldEdit.java

@@ -21,6 +21,7 @@
 
 import com.google.common.collect.ImmutableList;
 import com.sk89q.worldedit.WorldEdit;
+import com.sk89q.worldedit.cli.data.DataFile;
 import com.sk89q.worldedit.cli.data.FileRegistries;
 import com.sk89q.worldedit.cli.schematic.ClipboardWorld;
 import com.sk89q.worldedit.event.platform.CommandEvent;
@@ -100,7 +101,6 @@ private void setupPlatform() {
         WorldEdit.getInstance().getEventBus().post(new PlatformsRegisteredEvent());
 
         this.fileRegistries = new FileRegistries(this);
-        this.fileRegistries.loadDataFiles();
     }
 
     private void registerCommands() {
@@ -116,8 +116,11 @@ private void registerCommands() {
     }
 
     public void setupRegistries() {
+        this.fileRegistries.loadDataFiles();
+
         // Blocks
-        for (Map.Entry<String, FileRegistries.BlockManifest> manifestEntry : fileRegistries.getDataFile().blocks.entrySet()) {
+        BlockType.REGISTRY.clear();
+        for (Map.Entry<String, DataFile.BlockManifest> manifestEntry : fileRegistries.getDataFile().blocks().entrySet()) {
             if (BlockType.REGISTRY.get(manifestEntry.getKey()) == null) {
                 BlockType.REGISTRY.register(manifestEntry.getKey(), new BlockType(manifestEntry.getKey(), input -> {
                     ParserContext context = new ParserContext();
@@ -126,7 +129,7 @@ public void setupRegistries() {
                     context.setRestricted(false);
                     try {
                         FuzzyBlockState state = (FuzzyBlockState) WorldEdit.getInstance().getBlockFactory().parseFromInput(
-                            manifestEntry.getValue().defaultstate,
+                            manifestEntry.getValue().defaultState(),
                             context
                         ).toImmutableState();
                         BlockState defaultState = input.getBlockType().getAllStates().get(0);
@@ -144,30 +147,35 @@ public void setupRegistries() {
             }
         }
         // Items
-        for (String name : fileRegistries.getDataFile().items) {
+        ItemType.REGISTRY.clear();
+        for (String name : fileRegistries.getDataFile().items()) {
             if (ItemType.REGISTRY.get(name) == null) {
                 ItemType.REGISTRY.register(name, new ItemType(name));
             }
         }
         // Entities
-        for (String name : fileRegistries.getDataFile().entities) {
+        EntityType.REGISTRY.clear();
+        for (String name : fileRegistries.getDataFile().entities()) {
             if (EntityType.REGISTRY.get(name) == null) {
                 EntityType.REGISTRY.register(name, new EntityType(name));
             }
         }
         // Biomes
-        for (String name : fileRegistries.getDataFile().biomes) {
+        BiomeType.REGISTRY.clear();
+        for (String name : fileRegistries.getDataFile().biomes()) {
             if (BiomeType.REGISTRY.get(name) == null) {
                 BiomeType.REGISTRY.register(name, new BiomeType(name));
             }
         }
         // Tags
-        for (String name : fileRegistries.getDataFile().blocktags.keySet()) {
+        BlockCategory.REGISTRY.clear();
+        for (String name : fileRegistries.getDataFile().blockTags().keySet()) {
             if (BlockCategory.REGISTRY.get(name) == null) {
                 BlockCategory.REGISTRY.register(name, new BlockCategory(name));
             }
         }
-        for (String name : fileRegistries.getDataFile().itemtags.keySet()) {
+        ItemCategory.REGISTRY.clear();
+        for (String name : fileRegistries.getDataFile().itemTags().keySet()) {
             if (ItemCategory.REGISTRY.get(name) == null) {
                 ItemCategory.REGISTRY.register(name, new ItemCategory(name));
             }
@@ -185,7 +193,7 @@ public void onInitialized() {
             }
         }
 
-        this.commandSender = new CLICommandSender(this, LOGGER);
+        this.commandSender = new CLICommandSender(LOGGER);
         this.platform = new CLIPlatform(this);
         LOGGER.info("WorldEdit CLI (version " + getInternalVersion() + ") is loaded");
     }
@@ -333,6 +341,7 @@ public static void main(String[] args) {
                     try (ClipboardReader clipboardReader = format.getReader(Files.newInputStream(file.toPath(), StandardOpenOption.READ))) {
                         world = new ClipboardWorld(
                                 file,
+                                format,
                                 clipboardReader.read(),
                                 file.getName()
                         );

worldedit-cli/src/main/java/com/sk89q/worldedit/cli/data/DataFile.java

@@ -0,0 +1,40 @@
+/*
+ * WorldEdit, a Minecraft world manipulation toolkit
+ * Copyright (C) sk89q <http://www.sk89q.com>
+ * Copyright (C) WorldEdit team and contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.sk89q.worldedit.cli.data;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.util.List;
+import java.util.Map;
+
+public record DataFile(@SerializedName("itemtags") Map<String, List<String>> itemTags,
+                       @SerializedName("blocktags") Map<String, List<String>> blockTags,
+                       @SerializedName("entitytags") Map<String, List<String>> entityTags,
+                       List<String> items,
+                       List<String> entities,
+                       List<String> biomes,
+                       Map<String, BlockManifest> blocks) {
+
+    public record BlockManifest(@SerializedName("defaultstate") String defaultState, Map<String, BlockProperty> properties) {
+    }
+
+    public record BlockProperty(List<String> values, String type) {
+    }
+}

worldedit-cli/src/main/java/com/sk89q/worldedit/cli/data/FileRegistries.java

@@ -19,22 +19,25 @@
 
 package com.sk89q.worldedit.cli.data;
 
-import com.google.common.io.Resources;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.sk89q.worldedit.WorldEdit;
 import com.sk89q.worldedit.cli.CLIWorldEdit;
-import com.sk89q.worldedit.extension.platform.Capability;
-import com.sk89q.worldedit.util.io.ResourceLoader;
+import com.sk89q.worldedit.util.io.Closer;
 
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Map;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
 
 public class FileRegistries {
 
+    private static final int CLI_DATA_VERSION = 1;
+    private static final String DATA_FILE_DOWNLOAD_URL = "https://services.enginehub.org/cassette-deck/we-cli-data/";
+
     private final CLIWorldEdit app;
     private final Gson gson = new GsonBuilder().create();
 
@@ -45,36 +48,28 @@ public FileRegistries(CLIWorldEdit app) {
     }
 
     public void loadDataFiles() {
-        ResourceLoader resourceLoader = WorldEdit.getInstance().getPlatformManager().queryCapability(Capability.CONFIGURATION).getResourceLoader();
+        Path outputFolder = WorldEdit.getInstance().getWorkingDirectoryPath("cli-data");
+        Path checkPath = outputFolder.resolve(app.getPlatform().getDataVersion() + "_" + CLI_DATA_VERSION + ".json");
+
         try {
-            URL url = resourceLoader.getResource(FileRegistries.class, app.getPlatform().getDataVersion() + ".json");
-            this.dataFile = gson.fromJson(Resources.toString(url, StandardCharsets.UTF_8), DataFile.class);
+            Files.createDirectories(outputFolder);
+
+            if (!Files.exists(checkPath)) {
+                URL url = new URL(DATA_FILE_DOWNLOAD_URL + app.getPlatform().getDataVersion() + "/" + CLI_DATA_VERSION);
+
+                try (var stream = url.openStream()) {
+                    Files.copy(stream, checkPath);
+                }
+            }
+
+            this.dataFile = gson.fromJson(Files.readString(checkPath), DataFile.class);
         } catch (IOException e) {
-            throw new RuntimeException("The provided file is not compatible with this version of WorldEdit-CLI. Please update or report this.");
+            throw new RuntimeException("The provided file is not compatible with this version of WorldEdit-CLI. Please update or report this.", e);
         }
     }
 
     public DataFile getDataFile() {
         return this.dataFile;
     }
 
-    public static class BlockManifest {
-        public String defaultstate;
-        public Map<String, BlockProperty> properties;
-    }
-
-    public static class BlockProperty {
-        public List<String> values;
-        public String type;
-    }
-
-    public static class DataFile {
-        public Map<String, List<String>> itemtags;
-        public Map<String, List<String>> blocktags;
-        public Map<String, List<String>> entitytags;
-        public List<String> items;
-        public List<String> entities;
-        public List<String> biomes;
-        public Map<String, BlockManifest> blocks;
-    }
 }

worldedit-cli/src/main/java/com/sk89q/worldedit/cli/schematic/ClipboardWorld.java

@@ -29,7 +29,7 @@
 import com.sk89q.worldedit.entity.Entity;
 import com.sk89q.worldedit.extent.Extent;
 import com.sk89q.worldedit.extent.clipboard.Clipboard;
-import com.sk89q.worldedit.extent.clipboard.io.ClipboardFormats;
+import com.sk89q.worldedit.extent.clipboard.io.ClipboardFormat;
 import com.sk89q.worldedit.extent.clipboard.io.ClipboardWriter;
 import com.sk89q.worldedit.math.BlockVector3;
 import com.sk89q.worldedit.math.Vector3;
@@ -56,13 +56,15 @@
 public class ClipboardWorld extends AbstractWorld implements Clipboard, CLIWorld {
 
     private final File file;
+    private final ClipboardFormat format;
     private final Clipboard clipboard;
     private final String name;
 
     private boolean dirty = false;
 
-    public ClipboardWorld(File file, Clipboard clipboard, String name) {
+    public ClipboardWorld(File file, ClipboardFormat format, Clipboard clipboard, String name) {
         this.file = file;
+        this.format = format;
         this.clipboard = clipboard;
         this.name = name;
     }
@@ -74,7 +76,7 @@ public String getName() {
 
     @Override
     public String id() {
-        return getName().replace(" ", "_").toLowerCase(Locale.ROOT);
+        return name.replace(" ", "_").toLowerCase(Locale.ROOT);
     }
 
     @Override
@@ -208,7 +210,7 @@ public BlockVector3 getMinimumPoint() {
     @Override
     public void save(boolean force) {
         if (dirty || force) {
-            try (ClipboardWriter writer = ClipboardFormats.findByFile(file).getWriter(new FileOutputStream(file))) {
+            try (ClipboardWriter writer = format.getWriter(new FileOutputStream(file))) {
                 writer.write(this);
                 dirty = false;
             } catch (IOException e) {