From 906c416c99fef45617d6578941c22ac25db3c19a Mon Sep 17 00:00:00 2001 From: BuckarooBanzay Date: Thu, 11 Jan 2024 11:10:17 +0100 Subject: [PATCH] remove backend-handler construct, simplify --- api.lua | 37 ------ api.spec.lua | 6 +- backend_fs.lua | 44 ------- backend_patch.lua | 160 ----------------------- data.lua | 17 ++- functions.lua | 12 +- init.lua | 14 +- load.lua | 40 ++++++ mapgen.lua | 8 +- patch.lua | 87 ++++++++++++ backend_patch.spec.lua => patch.spec.lua | 3 +- readme.md | 16 +-- save.lua | 27 +++- 13 files changed, 185 insertions(+), 286 deletions(-) delete mode 100644 backend_fs.lua delete mode 100644 backend_patch.lua create mode 100644 load.lua create mode 100644 patch.lua rename backend_patch.spec.lua => patch.spec.lua (88%) diff --git a/api.lua b/api.lua index e2ea230..c9942e9 100644 --- a/api.lua +++ b/api.lua @@ -1,50 +1,13 @@ --- type => handler_def -local backend_handlers = {} - -function mapsync.register_backend_handler(name, handler) - -- default to no-op functions - handler.validate_config = handler.validate_config or function() end - handler.init = handler.init or function() end - - backend_handlers[name] = handler -end - -function mapsync.select_handler(backend_def) - return backend_handlers[backend_def.type] -end - --- type => handler_def -local data_backend_handlers = {} - -function mapsync.register_data_backend_handler(name, handler) - data_backend_handlers[name] = handler -end - -function mapsync.select_data_handler(data_backend_def) - return data_backend_handlers[data_backend_def.type] -end - -- name => backend_def local backends = {} -- register a map backend function mapsync.register_backend(name, backend_def) - local handler = mapsync.select_handler(backend_def) - if not handler then - error("unknown backend type: '" .. backend_def.type .. "' for backend '" .. name .. "'") - end - backend_def.name = name -- default to always-on backend if no selector specified backend_def.select = backend_def.select or function() return true end - -- validate config - handler.validate_config(backend_def) - - -- init backend def - handler.init(backend_def) - -- register backends[name] = backend_def end diff --git a/api.spec.lua b/api.spec.lua index 8cf5e2f..3ff3699 100644 --- a/api.spec.lua +++ b/api.spec.lua @@ -11,11 +11,9 @@ mtt.register("backend selection", function(callback) }) local backend_def = mapsync.select_backend({x=0, y=0, z=0}) - local handler = mapsync.select_handler(backend_def) - - assert(handler) assert(backend_def.name == "my-backend") - assert(handler.save_chunk(backend_def, {x=0, y=0, z=0})) + + assert(mapsync.save({x=0, y=0, z=0})) backend_def = mapsync.select_backend({x=0, y=10, z=0}) assert(not backend_def) diff --git a/backend_fs.lua b/backend_fs.lua deleted file mode 100644 index 401853c..0000000 --- a/backend_fs.lua +++ /dev/null @@ -1,44 +0,0 @@ -local global_env = ... - -local function get_path(prefix, chunk_pos) - return prefix .. "/chunk_" .. minetest.pos_to_string(chunk_pos) .. ".zip" -end - -mapsync.register_backend_handler("fs", { - validate_config = function(backend_def) - assert(type(backend_def.path) == "string") - end, - save_chunk = function(backend_def, chunk_pos) - return mapsync.serialize_chunk(chunk_pos, get_path(backend_def.path, chunk_pos)) - end, - - load_chunk = function(backend_def, chunk_pos, vmanip) - return mapsync.deserialize_chunk(chunk_pos, get_path(backend_def.path, chunk_pos), vmanip) - end, - - get_manifest = function(backend_def, chunk_pos) - return mapsync.get_manifest(get_path(backend_def.path, chunk_pos)) - end -}) - -mapsync.register_data_backend_handler("fs", { - validate_config = function(data_backend_def) - assert(type(data_backend_def.path) == "string") - end, - - save_data = function(data_backend_def, key, value) - local f = assert(global_env.io.open(data_backend_def.path .. "/" .. key .. ".lua", "w")) - f:write(minetest.serialize(value)) - f:close() - end, - - load_data = function(data_backend_def, key) - local f = global_env.io.open(data_backend_def.path .. "/" .. key .. ".lua", "r") - if not f then - return - end - local value = minetest.deserialize(f:read("*all")) - f:close() - return value - end -}) \ No newline at end of file diff --git a/backend_patch.lua b/backend_patch.lua deleted file mode 100644 index 535a36c..0000000 --- a/backend_patch.lua +++ /dev/null @@ -1,160 +0,0 @@ -local global_env = ... - -local function get_json_path(prefix, chunk_pos) - return prefix .. "/chunk_" .. minetest.pos_to_string(chunk_pos) .. ".json" -end - -local function get_path(prefix, chunk_pos) - return prefix .. "/chunk_" .. minetest.pos_to_string(chunk_pos) .. ".zip" -end - -mapsync.register_backend_handler("patch", { - validate_config = function(backend_def) - assert(type(backend_def.patch_path) == "string") - assert(type(backend_def.path) == "string") - end, - save_chunk = function(backend_def, chunk_pos) - local baseline_chunk = mapsync.parse_chunk(get_path(backend_def.path, chunk_pos)) - local filename = get_json_path(backend_def.patch_path, chunk_pos) - local f = global_env.io.open(filename, "w") - - local no_diff = true - mapsync.create_diff(baseline_chunk, chunk_pos, function(changed_node) - no_diff = false - f:write(minetest.write_json(changed_node) .. '\n') - end) - - f:close() - - if no_diff then - -- remove empty diff file - global_env.os.remove(filename) - end - - return true - end, - load_chunk = function(backend_def, chunk_pos, vmanip) - -- load baseline chunk (might be non-existent) - mapsync.deserialize_chunk(chunk_pos, get_path(backend_def.path, chunk_pos), vmanip) - - -- load diff if available - local f = io.open(get_json_path(backend_def.patch_path, chunk_pos), "r") - if not f then - -- no diff - return true - end - - local changed_nodes = {} - for line in f:lines() do - local changed_node = minetest.parse_json(line) - if changed_node then - table.insert(changed_nodes, changed_node) - end - end - f:close() - - -- apply diff - local success, msg = mapsync.apply_diff(chunk_pos, changed_nodes) - if not success then - return false, msg - end - - -- fix lighting - local mb_pos1, mb_pos2 = mapsync.get_mapblock_bounds_from_chunk(chunk_pos) - local pos1 = mapsync.get_node_bounds_from_mapblock(mb_pos1) - local _, pos2 = mapsync.get_node_bounds_from_mapblock(mb_pos2) - minetest.fix_light(pos1, pos2) - - return true - end, - get_manifest = function(backend_def, chunk_pos) - mapsync.get_manifest(get_path(backend_def.path, chunk_pos)) - end, - apply_patches = function(backend_def, callback, progress_callback) - -- load all chunks and save back to fs - local files = minetest.get_dir_list(backend_def.patch_path, false) - - -- collect all chunks - local chunk_pos_list = {} - for _, filename in ipairs(files) do - if string.match(filename, "^[chunk_(].*[).json]$") then - local pos_str = string.gsub(filename, "chunk_", "") - pos_str = string.gsub(pos_str, ".json", "") - - local chunk_pos = minetest.string_to_pos(pos_str) - table.insert(chunk_pos_list, chunk_pos) - end - end - - local emerge_count = 0 - for i, chunk_pos in ipairs(chunk_pos_list) do - mapsync.delete_chunk(chunk_pos) - mapsync.emerge_chunk(chunk_pos, function() - -- save emerged chunk - mapsync.serialize_chunk(chunk_pos, get_path(backend_def.path, chunk_pos)) - - -- remove patch file - local patch_path = get_json_path(backend_def.patch_path, chunk_pos) - global_env.os.remove(patch_path) - - -- report progress - if type(progress_callback) == "function" then - progress_callback(chunk_pos, #chunk_pos_list, i) - end - - emerge_count = emerge_count + 1 - if emerge_count < #chunk_pos_list then - --- not done yet - return - end - -- done - callback(emerge_count) - end) - end - end -}) - - -minetest.register_chatcommand("mapsync_apply_patches", { - description = "applies all the patches in given backend (defaults to the one at the current position)", - params = "[backend-name]", - privs = { mapsync = true }, - func = function(name, backend_name) - local player = minetest.get_player_by_name(name) - if not player then - return - end - - local pos = player:get_pos() - - local backend_def - if backend_name and backend_name ~= "" then - backend_def = mapsync.get_backend(backend_name) - if not backend_def then - return true, "Backend not found: '" .. backend_name .. "'" - end - else - local chunk_pos = mapsync.get_chunkpos(pos) - backend_def = mapsync.select_backend(chunk_pos) - if not backend_def then - return true, "Backend for current position not found" - end - end - - if backend_def.type ~= "patch" then - return true, "Backend type is not of type 'patch'" - end - - local handler = mapsync.select_handler(backend_def) - - -- apply patches back to shadowed backend - handler.apply_patches(backend_def, function(chunk_count) - minetest.chat_send_player(name, "Patching done with " .. chunk_count .. " chunk(s)") - end, function(chunk_pos, total_count, current_count) - minetest.chat_send_player(name, "Patched chunk " .. minetest.pos_to_string(chunk_pos) .. - " progress: " .. current_count .. "/" .. total_count) - end) - - return true, "Patching started" - end -}) \ No newline at end of file diff --git a/data.lua b/data.lua index 1c5c228..c2302aa 100644 --- a/data.lua +++ b/data.lua @@ -1,3 +1,4 @@ +local global_env = ... function mapsync.save_data(key, value) local data_backend_def = mapsync.get_data_backend() @@ -6,10 +7,9 @@ function mapsync.save_data(key, value) return end - local data_handler = mapsync.select_data_handler(data_backend_def) - if data_handler then - data_handler.save_data(data_backend_def, key, value) - end + local f = assert(global_env.io.open(data_backend_def.path .. "/" .. key .. ".lua", "w")) + f:write(minetest.serialize(value)) + f:close() end function mapsync.load_data(key) @@ -19,8 +19,11 @@ function mapsync.load_data(key) return end - local data_handler = mapsync.select_data_handler(data_backend_def) - if data_handler then - return data_handler.load_data(data_backend_def, key) + local f = global_env.io.open(data_backend_def.path .. "/" .. key .. ".lua", "r") + if not f then + return end + local value = minetest.deserialize(f:read("*all")) + f:close() + return value end \ No newline at end of file diff --git a/functions.lua b/functions.lua index 80cca3c..062df19 100644 --- a/functions.lua +++ b/functions.lua @@ -70,10 +70,8 @@ function mapsync.get_backend_chunk_mtime(chunk_pos) return end - local handler = mapsync.select_handler(backend_def) - -- get manifest - local manifest = handler.get_manifest(backend_def, chunk_pos) + local manifest = mapsync.get_manifest(mapsync.get_chunk_zip_path(backend_def.path, chunk_pos)) if not manifest then return end @@ -151,6 +149,14 @@ function mapsync.deep_compare(tbl1, tbl2) return false end +function mapsync.get_chunk_json_path(prefix, chunk_pos) + return prefix .. "/chunk_" .. minetest.pos_to_string(chunk_pos) .. ".json" +end + +function mapsync.get_chunk_zip_path(prefix, chunk_pos) + return prefix .. "/chunk_" .. minetest.pos_to_string(chunk_pos) .. ".zip" +end + -- https://gist.github.com/SafeteeWoW/080e784e5ebfda42cad486c58e6d26e4 -- license: zlib diff --git a/init.lua b/init.lua index f4b40ab..d8db6ce 100644 --- a/init.lua +++ b/init.lua @@ -26,33 +26,31 @@ end dofile(MP.."/api.lua") dofile(MP.."/privs.lua") --- backends -loadfile(MP.."/backend_fs.lua")(global_env) -loadfile(MP.."/backend_patch.lua")(global_env) - -- utilities / helpers dofile(MP.."/pos_iterator.lua") dofile(MP.."/encoding.lua") dofile(MP.."/serialize_mapblock.lua") dofile(MP.."/deserialize_mapblock.lua") dofile(MP.."/localize_nodeids.lua") +dofile(MP.."/functions.lua") -- diff / patch dofile(MP.."/create_diff.lua") dofile(MP.."/apply_diff.lua") +loadfile(MP.."/patch.lua")(global_env) -- save/load dofile(MP.."/auto_save.lua") dofile(MP.."/auto_update.lua") -dofile(MP.."/save.lua") +loadfile(MP.."/save.lua")(global_env) +loadfile(MP.."/data.lua")(global_env) +dofile(MP.."/load.lua") dofile(MP.."/mapgen.lua") -dofile(MP.."/data.lua") -- hud stuff dofile(MP.."/hud.lua") -- pass on global env (secure/insecure) -loadfile(MP.."/functions.lua")(global_env) loadfile(MP.."/serialize_chunk.lua")(global_env) loadfile(MP.."/parse_chunk.lua")(global_env) loadfile(MP.."/deserialize_chunk.lua")(global_env) @@ -72,7 +70,7 @@ if minetest.get_modpath("mtt") and mtt.enabled then dofile(MP.."/crypto.spec.lua") dofile(MP.."/data.spec.lua") dofile(MP.."/diff.spec.lua") - dofile(MP.."/backend_patch.spec.lua") + dofile(MP.."/patch.spec.lua") dofile(MP.."/api.spec.lua") dofile(MP.."/serialize_chunk.spec.lua") end \ No newline at end of file diff --git a/load.lua b/load.lua new file mode 100644 index 0000000..98ccbc2 --- /dev/null +++ b/load.lua @@ -0,0 +1,40 @@ + +function mapsync.load(chunk_pos, vmanip) + local backend_def = mapsync.select_backend(chunk_pos) + if not backend_def then + return true, "No backend available" + end + + -- load baseline chunk (might be non-existent) + mapsync.deserialize_chunk(chunk_pos, mapsync.get_chunk_zip_path(backend_def.path, chunk_pos), vmanip) + + if backend_def.patch_path then + -- load diff + local f = io.open(mapsync.get_chunk_json_path(backend_def.patch_path, chunk_pos), "r") + if not f then + -- no diff + return true + end + + local changed_nodes = {} + for line in f:lines() do + local changed_node = minetest.parse_json(line) + if changed_node then + table.insert(changed_nodes, changed_node) + end + end + f:close() + + -- apply diff + local success, msg = mapsync.apply_diff(chunk_pos, changed_nodes) + if not success then + return false, msg + end + + -- fix lighting + local mb_pos1, mb_pos2 = mapsync.get_mapblock_bounds_from_chunk(chunk_pos) + local pos1 = mapsync.get_node_bounds_from_mapblock(mb_pos1) + local _, pos2 = mapsync.get_node_bounds_from_mapblock(mb_pos2) + minetest.fix_light(pos1, pos2) + end +end \ No newline at end of file diff --git a/mapgen.lua b/mapgen.lua index f220fd7..04b2043 100644 --- a/mapgen.lua +++ b/mapgen.lua @@ -1,15 +1,9 @@ minetest.register_on_generated(function(minp) local chunk_pos = mapsync.get_chunkpos(minp) - local backend_def = mapsync.select_backend(chunk_pos) - if not backend_def then - return - end - - local handler = mapsync.select_handler(backend_def) local t1 = minetest.get_us_time() local vmanip = minetest.get_mapgen_object("voxelmanip") - handler.load_chunk(backend_def, chunk_pos, vmanip) + mapsync.load(chunk_pos, vmanip) local t2 = minetest.get_us_time() local micros = t2 - t1 diff --git a/patch.lua b/patch.lua new file mode 100644 index 0000000..af6b80b --- /dev/null +++ b/patch.lua @@ -0,0 +1,87 @@ +local global_env = ... + +function mapsync.apply_patches(backend_def, callback, progress_callback) + -- load all chunks and save back to fs + local files = minetest.get_dir_list(backend_def.patch_path, false) + + -- collect all chunks + local chunk_pos_list = {} + for _, filename in ipairs(files) do + if string.match(filename, "^[chunk_(].*[).json]$") then + local pos_str = string.gsub(filename, "chunk_", "") + pos_str = string.gsub(pos_str, ".json", "") + + local chunk_pos = minetest.string_to_pos(pos_str) + table.insert(chunk_pos_list, chunk_pos) + end + end + + local emerge_count = 0 + for i, chunk_pos in ipairs(chunk_pos_list) do + mapsync.delete_chunk(chunk_pos) + mapsync.emerge_chunk(chunk_pos, function() + -- save emerged chunk + mapsync.serialize_chunk(chunk_pos, mapsync.get_chunk_zip_path(backend_def.path, chunk_pos)) + + -- remove patch file + local patch_path = mapsync.get_chunk_json_path(backend_def.patch_path, chunk_pos) + global_env.os.remove(patch_path) + + -- report progress + if type(progress_callback) == "function" then + progress_callback(chunk_pos, #chunk_pos_list, i) + end + + emerge_count = emerge_count + 1 + if emerge_count < #chunk_pos_list then + --- not done yet + return + end + -- done + callback(emerge_count) + end) + end +end + + +minetest.register_chatcommand("mapsync_apply_patches", { + description = "applies all the patches in given backend (defaults to the one at the current position)", + params = "[backend-name]", + privs = { mapsync = true }, + func = function(name, backend_name) + local player = minetest.get_player_by_name(name) + if not player then + return + end + + local pos = player:get_pos() + + local backend_def + if backend_name and backend_name ~= "" then + backend_def = mapsync.get_backend(backend_name) + if not backend_def then + return true, "Backend not found: '" .. backend_name .. "'" + end + else + local chunk_pos = mapsync.get_chunkpos(pos) + backend_def = mapsync.select_backend(chunk_pos) + if not backend_def then + return true, "Backend for current position not found" + end + end + + if backend_def.type ~= "patch" then + return true, "Backend type is not of type 'patch'" + end + + -- apply patches back to shadowed backend + mapsync.apply_patches(backend_def, function(chunk_count) + minetest.chat_send_player(name, "Patching done with " .. chunk_count .. " chunk(s)") + end, function(chunk_pos, total_count, current_count) + minetest.chat_send_player(name, "Patched chunk " .. minetest.pos_to_string(chunk_pos) .. + " progress: " .. current_count .. "/" .. total_count) + end) + + return true, "Patching started" + end +}) \ No newline at end of file diff --git a/backend_patch.spec.lua b/patch.spec.lua similarity index 88% rename from backend_patch.spec.lua rename to patch.spec.lua index ba73c44..7c1de9a 100644 --- a/backend_patch.spec.lua +++ b/patch.spec.lua @@ -34,10 +34,9 @@ mtt.register("patch backend", function(callback) -- get patch handler local patch_backend_def = mapsync.get_backend("my-patched-backend") - local patch_handler = mapsync.select_handler(patch_backend_def) -- apply patches back to shadowed backend - patch_handler.apply_patches(patch_backend_def, function(chunk_count) + mapsync.apply_patches(patch_backend_def, function(chunk_count) assert(chunk_count == 1) -- cleanup mapsync.unregister_backend("my-patched-backend") diff --git a/readme.md b/readme.md index 69029e6..586797d 100644 --- a/readme.md +++ b/readme.md @@ -9,17 +9,12 @@ mapsync mod Synchronize the ingame map with a lua-backend -Supported lua-backends: -* `fs` local filesystem (can be in a world- or mod-folder) - -Planned backends: -* `http` http/webdav backend - Features: -* Auto-update chunks if a newer version on the backend is found +* Writes and saves maps to/from zip-files +* Auto-updates chunks if a newer version on the backend is found +* Patching/Merging support (with diff files) Planned features: -* Diffing/Merging/Applying changes from multiple sources (git merges for example) * `placeholder` support # Use case @@ -35,12 +30,11 @@ Create a new mod (or use an existing one) and add the backend-registration: For storage in a world-folder: ```lua local path = minetest.get_worldpath() .. "/mymap" --- ensure the path exists +-- ensure that the path exists minetest.mkdir(path) -- register the backend mapsync.register_backend("my-backend", { - type = "fs", path = path }) ``` @@ -50,7 +44,6 @@ To store it in a mod-folder: -- store and load the map in the "map" folder of the "my-mod" mod: -- NOTE: the `mapsync` mod needs to be in the `secure.trusted_mods` setting for write-access mapsync.register_backend("my-backend", { - type = "fs", path = minetest.get_modpath("my-mod") .. "/map" }) ``` @@ -64,7 +57,6 @@ The saved chunks will now automatically be loaded if the destination area is gen The backend can implement the `select` function to only synchronize a subset of the world: ```lua mapsync.register_backend("my-backend", { - type = "fs", path = minetest.get_worldpath() .. "/mymap", select = function(chunk_pos) -- only save/load chunks between the 10 and -10 y-chunk layer diff --git a/save.lua b/save.lua index daec507..ab0b662 100644 --- a/save.lua +++ b/save.lua @@ -1,3 +1,4 @@ +local global_env = ... local function save_worker(ctx) local chunk_pos = ctx.iterator() @@ -70,6 +71,28 @@ function mapsync.save(chunk_pos) return true, "No backend available" end - local handler = mapsync.select_handler(backend_def) - return handler.save_chunk(backend_def, chunk_pos) + if not backend_def.patch_path then + -- direct save + return mapsync.serialize_chunk(chunk_pos, mapsync.get_chunk_zip_path(backend_def.path, chunk_pos)) + else + -- create patch file + local baseline_chunk = mapsync.parse_chunk(mapsync.get_chunk_zip_path(backend_def.path, chunk_pos)) + local filename = mapsync.get_chunk_json_path(backend_def.patch_path, chunk_pos) + local f = global_env.io.open(filename, "w") + + local no_diff = true + mapsync.create_diff(baseline_chunk, chunk_pos, function(changed_node) + no_diff = false + f:write(minetest.write_json(changed_node) .. '\n') + end) + + f:close() + + if no_diff then + -- remove empty diff file + global_env.os.remove(filename) + end + + return true + end end \ No newline at end of file