From 9698e8fabe2616fd3d1085a3e085f48bc5a29a75 Mon Sep 17 00:00:00 2001 From: Buckaroo Banzai <39065740+BuckarooBanzay@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:43:00 +0100 Subject: [PATCH] Upload overhaul (#400) * overhaul filebrowser upload * uploaded file rows * remove external filebrowser * move log retention var to Config * cleanup * increase chunk size * progress bar * luanti image * fix corrupt zip download * fix mtui database restore * update latest engine image version * nav fix * add backup/restore page stub * backup/restore wip * replace maintenance page with backup --------- Co-authored-by: BuckarooBanzay --- app/database.go | 12 +- docker-compose.yml | 14 -- jobs/log_cleanup.go | 8 +- public/js/components/NavBar.js | 6 +- .../pages/administration/BackupRestore.js | 127 ++++++++++++++++++ .../pages/administration/Maintenance.js | 70 ---------- .../pages/administration/UISettings.js | 44 +++--- .../pages/filebrowser/Filebrowser.js | 124 +++++------------ public/js/routes.js | 8 +- public/js/service/stats.js | 1 - public/js/service/uploader.js | 8 +- types/config.go | 6 +- types/services.go | 11 +- web/filebrowser_download.go | 4 - web/maintenance.go | 3 +- web/setup.go | 23 ---- web/stats.go | 6 +- 17 files changed, 222 insertions(+), 253 deletions(-) create mode 100644 public/js/components/pages/administration/BackupRestore.js delete mode 100644 public/js/components/pages/administration/Maintenance.js diff --git a/app/database.go b/app/database.go index 1e00dee8..175e34a1 100644 --- a/app/database.go +++ b/app/database.go @@ -106,7 +106,17 @@ func (a *App) DetachDatabase() error { a.ModManager = nil a.Mail = nil a.Repos = nil - err := a.DB.Close() + gdb, err := a.G.DB() + if err != nil { + return fmt.Errorf("could not get gorm database: %v", err) + } + + err = gdb.Close() + if err != nil { + return fmt.Errorf("could not close gorm database: %v", err) + } + + err = a.DB.Close() if err != nil { return fmt.Errorf("could not close database: %v", err) } diff --git a/docker-compose.yml b/docker-compose.yml index b55ef2ca..24f82a9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,6 @@ services: SERVER_NAME: "dev-server" DEFAULT_THEME: "darkly" ENABLE_FEATURES: "shell,luashell,minetest_config,docker,modmanagement,signup,chat,minetest_web" - FILEBROWSER_URL: "http://filebrowser/" INSTALL_MTUI_MOD: "true" MINETEST_CONFIG: "/world/minetest.conf" GEOIP_API: "https://hosting.minetest.ch/api/geoip" @@ -40,21 +39,8 @@ services: - "/var/run/docker.sock:/var/run/docker.sock:ro" working_dir: /app command: ["go", "run", "."] - - filebrowser: - image: filebrowser/filebrowser:v2.31.2 - ports: - - 8081:80 - environment: - FB_DATABASE: /database/filebrowser.db - FB_BASEURL: /filebrowser - FB_NOAUTH: "true" - volumes: - - world_dir:/srv - - filebrowser_db:/database volumes: go_cache: {} go_dir: {} world_dir: {} - filebrowser_db: {} \ No newline at end of file diff --git a/jobs/log_cleanup.go b/jobs/log_cleanup.go index 1240bf5f..89f85355 100644 --- a/jobs/log_cleanup.go +++ b/jobs/log_cleanup.go @@ -3,18 +3,16 @@ package jobs import ( "fmt" "mtui/app" - "os" "time" ) func logCleanup(a *app.App) { - log_retention_str := os.Getenv("LOG_RETENTION") log_retention := time.Hour * 24 * 7 // 7 days default log retention - if log_retention_str != "" { + if a.Config.LogRetention != "" { var err error - log_retention, err = time.ParseDuration(log_retention_str) + log_retention, err = time.ParseDuration(a.Config.LogRetention) if err != nil { - fmt.Printf("Log retention parsing of '%s' failed: %v, defaulting to 7 days\n", log_retention_str, err) + fmt.Printf("Log retention parsing of '%s' failed: %v, defaulting to 7 days\n", a.Config.LogRetention, err) log_retention = time.Hour * 24 * 7 } } diff --git a/public/js/components/NavBar.js b/public/js/components/NavBar.js index 0a5b55a2..5e07fc02 100644 --- a/public/js/components/NavBar.js +++ b/public/js/components/NavBar.js @@ -169,14 +169,14 @@ export default { Mediaserver -
  • +
  • Restart conditions
  • - - Maintenance + + Backup/Restore
    • diff --git a/public/js/components/pages/administration/BackupRestore.js b/public/js/components/pages/administration/BackupRestore.js new file mode 100644 index 00000000..5f11b465 --- /dev/null +++ b/public/js/components/pages/administration/BackupRestore.js @@ -0,0 +1,127 @@ +import { get_maintenance, get_stats } from "../../../service/stats.js"; +import { enable_maintenance, disable_maintenance } from "../../../api/maintenance.js"; +import { engine } from "../../../service/service.js"; +import { upload_chunked } from "../../../service/uploader.js"; +import { unzip, remove } from "../../../api/filebrowser.js"; + +import DefaultLayout from "../../layouts/DefaultLayout.js"; +import { START } from "../../Breadcrumb.js"; + +export default { + components: { + "default-layout": DefaultLayout + }, + data: function() { + return { + breadcrumb: [START, { + name: "Backup/Restore", + icon: "upload", + link: "/backup" + }], + restore_active: false, + restore_message: "", + restore_progress: 0 + }; + }, + computed: { + maintenance: get_maintenance, + is_engine_running: () => engine.is_running() + }, + methods: { + enable_maintenance: async function() { + await enable_maintenance(); + get_stats(); + }, + disable_maintenance: async function() { + await disable_maintenance(); + window.location.reload(); + }, + restore: async function() { + const file = this.$refs.input_upload.files[0]; + if (!file) { + return; + } + + this.restore_message = "Starting to upload archive"; + this.restore_active = true; + + await upload_chunked("/", "restore.zip", file, progress => { + this.restore_progress = progress; + this.restore_message = `Uploading: ${Math.floor(progress*100)}% done`; + }); + + this.restore_message = "Unzipping archive..."; + await unzip("/restore.zip"); + + this.restore_message = "Removing temporary archive"; + await remove("/restore.zip"); + + this.restore_active = false; + + await this.disable_maintenance(); + } + }, + template: /*html*/` + +
    +
    +
    +
    + Download backup +
    +
    + + + Download world-backup as zip-file + +
    +
    +
    +
    +
    +
    + Restore from backup +
    +
    +
    + + The maintenance mode must be enabled and all the services stopped to restore from a backup +
    + +
    + + The minetest engine is still running, please stop it to enable the maintenance mode +
    + +
    + + Warning: All existing world-data will be overwritten by a backup-restore! +
    + +
    + + + + +
    + +
    +
    + {{restore_message}} +
    +
    +
    +
    +
    +
    +     + ` +}; \ No newline at end of file diff --git a/public/js/components/pages/administration/Maintenance.js b/public/js/components/pages/administration/Maintenance.js deleted file mode 100644 index ffd76b8c..00000000 --- a/public/js/components/pages/administration/Maintenance.js +++ /dev/null @@ -1,70 +0,0 @@ -import { get_maintenance, get_stats } from "../../../service/stats.js"; -import { enable_maintenance, disable_maintenance } from "../../../api/maintenance.js"; -import { engine } from "../../../service/service.js"; -import DefaultLayout from "../../layouts/DefaultLayout.js"; -import { START, ADMINISTRATION } from "../../Breadcrumb.js"; - -export default { - data: function() { - return { - breadcrumb: [START, ADMINISTRATION, { - icon: "wrench", - name: "Maintenance", - link: "/maintenance" - }] - }; - }, - components: { - "default-layout": DefaultLayout - }, - computed: { - maintenance: get_maintenance, - is_engine_running: () => engine.is_running() - }, - methods: { - enable_maintenance: function() { - enable_maintenance() - .then(() => get_stats()); - }, - disable_maintenance: function() { - disable_maintenance() - .then(() => window.location.reload()); - } - }, - template: /*html*/` - -

    Maintenance

    - - - - - - - - - - - -
    Maintenance mode - Disabled - Enabled -
    Actions - - -
    -
    - - The maintenance mode shuts down any database access, it allows you to create and download consistent backups -
    -
    - - The minetest engine is still running, please stop it to enable the maintenance mode -
    -     - ` -}; \ No newline at end of file diff --git a/public/js/components/pages/administration/UISettings.js b/public/js/components/pages/administration/UISettings.js index ee696d26..9373ccd6 100644 --- a/public/js/components/pages/administration/UISettings.js +++ b/public/js/components/pages/administration/UISettings.js @@ -37,26 +37,30 @@ export default { template: /*html*/` - - - - - - - - - - + + + + + + + + + + + + + +
    SettingValueAction
    - Theme - - - - - Save - -
    SettingValueAction
    + Theme + + + + + Save + +
    ` diff --git a/public/js/components/pages/filebrowser/Filebrowser.js b/public/js/components/pages/filebrowser/Filebrowser.js index 9ddee045..932720a7 100644 --- a/public/js/components/pages/filebrowser/Filebrowser.js +++ b/public/js/components/pages/filebrowser/Filebrowser.js @@ -1,14 +1,9 @@ import DefaultLayout from "../../layouts/DefaultLayout.js"; import { browse, - get_zip_url, - get_targz_url, get_download_url, mkdir, remove, - upload, - upload_zip, - upload_targz, rename } from "../../../api/filebrowser.js"; import { upload_chunked } from "../../../service/uploader.js"; @@ -16,7 +11,6 @@ import format_size from "../../../util/format_size.js"; import format_time from "../../../util/format_time.js"; import { START, FILEBROWSER } from "../../Breadcrumb.js"; import { can_edit } from "./common.js"; -import { get_filebrowser_enabled } from "../../../service/stats.js"; export default { props: ["pathMatch"], @@ -26,64 +20,38 @@ export default { data: function() { return { result: null, - mkfile_name: "", + mkdir_name: "", move_name: "", move_target: "", - upload_busy: false, - upload_archive_busy: false, - prepare_delete: null + prepare_delete: null, + upload_progress: {} }; }, methods: { format_size, format_time, - get_zip_url, - get_targz_url, get_download_url, mkdir: function() { - mkdir(this.result.dir + "/" + this.mkfile_name) + mkdir(this.result.dir + "/" + this.mkdir_name) .then(() => this.browse_dir()) - .then(() => this.mkfile_name = ""); - }, - mkfile: function() { - upload(this.result.dir + "/" + this.mkfile_name, "") - .then(() => this.browse_dir()) - .then(() => this.mkfile_name = ""); + .then(() => this.mkdir_name = ""); }, upload: function() { const files = Array.from(this.$refs.input_upload.files); - this.upload_busy = true; - const promises = files.map(file => upload_chunked(this.result.dir, file.name, file)); - - Promise.all(promises).then(() => { - this.$refs.input_upload.value = null; - this.upload_busy = false; - this.browse_dir(); + files.forEach(file => { + upload_chunked(this.result.dir, file.name, file, progress => { + this.upload_progress[file.name] = { + progress, + name: file.name, + size: file.size + }; + }).then(() => { + delete this.upload_progress[file.name]; + this.browse_dir(); + }); }); - }, - upload_archive: function() { - if (this.$refs.input_upload_archive.files.length == 0) { - return; - } - this.upload_archive_busy = true; - const file = this.$refs.input_upload_archive.files[0]; - let upload_fn = null; - if (file.name.endsWith(".zip")) { - upload_fn = upload_zip; - } else if (file.name.endsWith(".tar.gz")) { - upload_fn = upload_targz; - } else { - this.$refs.input_upload_archive.value = null; - this.upload_archive_busy = false; - return; - } - upload_fn(this.result.dir, file) - .then(() => { - this.$refs.input_upload_archive.value = null; - this.upload_archive_busy = false; - this.browse_dir(); - }); + this.$refs.input_upload.value = null; }, confirm_delete: function() { remove(this.result.dir + "/" + this.prepare_delete) @@ -109,7 +77,6 @@ export default { }); }, can_edit: can_edit, - filebrowser_enabled: get_filebrowser_enabled, is_json_profile: function(filename) { return filename.match(/^profile-.*.json$/); }, @@ -162,65 +129,26 @@ export default { }, template: /*html*/` -
    -
    -
    - - Note: To upload larger files use the dedicated filebrowser interface and enable the maintenance mode for database files -
    -
    -
    - - -
    -
    +
    -
    -
    -
    -
    - -
    -
    - - - Download - - - - zip - - - - tar.gz - -
    @@ -232,6 +160,18 @@ export default { + + + +
    + + {{name}} ({{Math.floor(entry.progress * 100)}} % / {{format_size(entry.size)}}) + +
    +
    +
    +
    +
    diff --git a/public/js/routes.js b/public/js/routes.js index 1a425986..33c7e9b9 100644 --- a/public/js/routes.js +++ b/public/js/routes.js @@ -19,7 +19,6 @@ import OauthApps from './components/pages/oauth/OauthApps.js'; import OauthAppEdit from './components/pages/oauth/OauthAppEdit.js'; import MinetestConfig from './components/pages/administration/MinetestConfig.js'; import UISettings from './components/pages/administration/UISettings.js'; -import Maintenance from './components/pages/administration/Maintenance.js'; import Filebrowser from './components/pages/filebrowser/Filebrowser.js'; import FileEditPage from './components/pages/filebrowser/FileEditPage.js'; import Signup from './components/pages/Signup.js'; @@ -40,13 +39,11 @@ import Mesecons from './components/pages/Mesecons.js'; import Luacontroller from './components/pages/Luacontroller.js'; import Play from './components/pages/Play.js'; import RestartConditions from './components/pages/administration/RestartConditions.js'; +import BackupRestore from './components/pages/administration/BackupRestore.js'; export default [{ path: "/", component: Start, meta: { maintenance_page: true } -}, { - path: "/maintenance", component: Maintenance, - meta: { requiredPriv: "server", maintenance_page: true } }, { path: "/restart-conditions", component: RestartConditions, meta: { requiredPriv: "server" } @@ -100,6 +97,9 @@ export default [{ }, { path: "/mods", component: Mods, meta: { requiredPriv: "server" } +}, { + path: "/backup", component: BackupRestore, + meta: { requiredPriv: "server", maintenance_page: true } }, { path: "/cdb/browse", component: ContentBrowse, meta: { requiredPriv: "server" } diff --git a/public/js/service/stats.js b/public/js/service/stats.js index 302b12e6..fadde270 100644 --- a/public/js/service/stats.js +++ b/public/js/service/stats.js @@ -19,4 +19,3 @@ export const stop_polling = () => clearInterval(handle); export const get_player_count = () => store.player_count; export const get_players = () => store.players; export const get_maintenance = () => store.maintenance; -export const get_filebrowser_enabled = () => store.filebrowser_enabled; \ No newline at end of file diff --git a/public/js/service/uploader.js b/public/js/service/uploader.js index 192ae515..24f74679 100644 --- a/public/js/service/uploader.js +++ b/public/js/service/uploader.js @@ -1,6 +1,6 @@ import { append, browse, remove, rename } from "../api/filebrowser.js"; -export async function upload_chunked(dir, filename, data) { +export async function upload_chunked(dir, filename, data, progress_callback) { // temp filename to upload to const tmpfilename = filename + ".part"; @@ -12,9 +12,13 @@ export async function upload_chunked(dir, filename, data) { let offset = 0; do { - const chunksize = Math.min(data.size - offset, 1000*1000); // 1 mb chunks + const chunksize = Math.min(data.size - offset, 1000*1000*2); // 2 mb chunks await append(dir + "/" + tmpfilename, data.slice(offset, offset + chunksize)); offset += chunksize; + + if (typeof(progress_callback) == "function") { + progress_callback(offset / data.size); // 0...1 + } } while (offset < data.size); await rename(dir + "/" + tmpfilename, dir + "/" + filename); diff --git a/types/config.go b/types/config.go index 3bb4a62b..a64e85de 100644 --- a/types/config.go +++ b/types/config.go @@ -18,10 +18,9 @@ type Config struct { Webdev bool Servername string EnabledFeatures []string - FilebrowserURL string - FilebrowserProxyPath string InstallMtuiMod bool AutoReconfigureMods bool + LogRetention string LogStreamURL string LogStreamAuthorization string MinetestConfig string @@ -50,10 +49,9 @@ func NewConfig(world_dir string) *Config { Webdev: os.Getenv("WEBDEV") == "true", Servername: os.Getenv("SERVER_NAME"), EnabledFeatures: strings.Split(os.Getenv("ENABLE_FEATURES"), ","), - FilebrowserURL: os.Getenv("FILEBROWSER_URL"), - FilebrowserProxyPath: os.Getenv("FILEBROWSER_PROXY_PATH"), InstallMtuiMod: os.Getenv("INSTALL_MTUI_MOD") == "true", AutoReconfigureMods: os.Getenv("AUTORECONFIGURE_MODS") == "true", + LogRetention: os.Getenv("LOG_RETENTION"), LogStreamURL: os.Getenv("LOG_STREAM_URL"), LogStreamAuthorization: os.Getenv("LOG_STREAM_AUTHORIZATION"), MinetestConfig: os.Getenv("MINETEST_CONFIG"), diff --git a/types/services.go b/types/services.go index b124fafd..46cbdbb5 100644 --- a/types/services.go +++ b/types/services.go @@ -1,14 +1,15 @@ package types var EngineServiceImages = map[string]string{ - "5.6.0": "registry.gitlab.com/minetest/minetest/server:5.6.0", - "5.7.0": "registry.gitlab.com/minetest/minetest/server:5.7.0", - "5.8.0": "ghcr.io/minetest-hosting/minetest-docker:5.8.0", - "5.9.0": "ghcr.io/minetest/minetest:5.9.0", + "5.6.0": "registry.gitlab.com/minetest/minetest/server:5.6.0", + "5.7.0": "registry.gitlab.com/minetest/minetest/server:5.7.0", + "5.8.0": "ghcr.io/minetest-hosting/minetest-docker:5.8.0", + "5.9.0": "ghcr.io/minetest/minetest:5.9.0", + "5.10.0": "ghcr.io/minetest/minetest:5.10.0", } // for auto install -var EngineServiceLatest = "5.9.0" +var EngineServiceLatest = "5.10.0" var MatterbridgeServiceImages = map[string]string{ "1.26.0": "42wim/matterbridge:1.26.0", diff --git a/web/filebrowser_download.go b/web/filebrowser_download.go index 5a705788..dd84e66a 100644 --- a/web/filebrowser_download.go +++ b/web/filebrowser_download.go @@ -1,7 +1,6 @@ package web import ( - "archive/zip" "fmt" "io" "mtui/app" @@ -82,9 +81,6 @@ func (a *Api) DownloadZip(w http.ResponseWriter, r *http.Request, claims *types. w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", zipfilename)) w.Header().Set("Content-Type", "application/zip") - zw := zip.NewWriter(w) - defer zw.Close() - count, err := a.app.StreamZip(absdir, w, nil) if err != nil { SendError(w, 500, err) diff --git a/web/maintenance.go b/web/maintenance.go index ad019f85..a08e73f5 100644 --- a/web/maintenance.go +++ b/web/maintenance.go @@ -18,7 +18,6 @@ func (a *Api) EnableMaintenanceMode(w http.ResponseWriter, r *http.Request, c *t if a.app.MaintenanceMode.Load() { SendError(w, 500, fmt.Errorf("already in maintenance mode")) } - a.app.MaintenanceMode.Store(true) // create log entry a.app.CreateUILogEntry(&types.Log{ @@ -27,6 +26,8 @@ func (a *Api) EnableMaintenanceMode(w http.ResponseWriter, r *http.Request, c *t Message: fmt.Sprintf("User '%s' enables the maintenance mode", c.Username), }, r) + a.app.MaintenanceMode.Store(true) + // clear current stats current_stats.Store(nil) // detach database diff --git a/web/setup.go b/web/setup.go index f3e59a41..601c5e9f 100644 --- a/web/setup.go +++ b/web/setup.go @@ -1,13 +1,10 @@ package web import ( - "fmt" "mtui/app" "mtui/public" "mtui/types" "net/http" - "net/http/httputil" - "net/url" "os" "time" @@ -31,26 +28,6 @@ func Setup(a *app.App) error { return err } - if a.Config.FilebrowserURL != "" { - // enable filebrowser access with "server" priv - remote, err := url.Parse(a.Config.FilebrowserURL) - if err != nil { - return err - } - - handler := func(p *httputil.ReverseProxy) func(http.ResponseWriter, *http.Request) { - return api.SecurePriv("server", func(w http.ResponseWriter, r *http.Request, c *types.Claims) { - r.Host = remote.Host - // prepend proxy path - r.URL.Path = fmt.Sprintf("%s%s", a.Config.FilebrowserProxyPath, r.URL.Path) - p.ServeHTTP(w, r) - }) - } - - proxy := httputil.NewSingleHostReverseProxy(remote) - r.PathPrefix("/filebrowser/").HandlerFunc(handler(proxy)) - } - // always on api r.HandleFunc("/api/maintenance", api.SecurePriv(types.PRIV_SERVER, api.GetMaintenanceMode)).Methods(http.MethodGet) r.HandleFunc("/api/maintenance", api.SecurePriv(types.PRIV_SERVER, api.EnableMaintenanceMode)).Methods(http.MethodPut) diff --git a/web/stats.go b/web/stats.go index 2b40c830..7c6eae3a 100644 --- a/web/stats.go +++ b/web/stats.go @@ -30,15 +30,13 @@ func (a *Api) StatsEventListener(c chan *bridge.CommandResponse) { type StatResponse struct { *command.StatsCommand - Maintenance bool `json:"maintenance"` - FilebrowserEnabled bool `json:"filebrowser_enabled"` + Maintenance bool `json:"maintenance"` } func (a *Api) GetStats(w http.ResponseWriter, r *http.Request, claims *types.Claims) { sc := &StatResponse{ - StatsCommand: &command.StatsCommand{}, - FilebrowserEnabled: a.app.Config.FilebrowserURL != "", + StatsCommand: &command.StatsCommand{}, } sc.Maintenance = a.app.MaintenanceMode.Load()