From 903411cde00c59a67b51b2284c2b2dad309a55c5 Mon Sep 17 00:00:00 2001 From: Code by Ben Date: Mon, 16 Dec 2024 11:43:48 +1100 Subject: [PATCH] fix TextFiles to remove duplicate diz and readme texts used for the same uuid record. --- flags/flags.go | 4 ++- internal/config/fixer.go | 4 ++- internal/config/repair.go | 55 ++++++++++++++++++++++++++++++++++++--- model/model.go | 10 +++++++ 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/flags/flags.go b/flags/flags.go index 2a243eb1..9b7745ff 100644 --- a/flags/flags.go +++ b/flags/flags.go @@ -10,6 +10,7 @@ import ( "runtime" "strconv" "strings" + "time" "github.com/Defacto2/server/internal/config" "github.com/carlmjohnson/versioninfo" @@ -66,7 +67,8 @@ func Fix(c *config.Config) *cli.Command { Usage: "fix the database and assets", Description: "Fix the database entries and file assets by running scans and checks.", Action: func(_ *cli.Context) error { - if err := c.Fixer(); err != nil { + d := time.Now() + if err := c.Fixer(d); err != nil { return fmt.Errorf("command fix: %w", err) } return nil diff --git a/internal/config/fixer.go b/internal/config/fixer.go index 9d66ac01..29ac2d4d 100644 --- a/internal/config/fixer.go +++ b/internal/config/fixer.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "strings" + "time" "github.com/Defacto2/helper" "github.com/Defacto2/server/internal/command" @@ -16,7 +17,7 @@ import ( ) // Fixer is used to fix any known issues with the file assets and the database entries. -func (c Config) Fixer() error { +func (c Config) Fixer(d time.Time) error { logger := zaplog.Timestamp().Sugar() db, err := postgres.Open() if err != nil { @@ -42,6 +43,7 @@ func (c Config) Fixer() error { c.repairer(ctx, db) c.sanityChecks(ctx) SanityTmpDir() + logger.Infof("Fixer completed in %.1fs", time.Since(d).Seconds()) return nil } diff --git a/internal/config/repair.go b/internal/config/repair.go index 4a4a857d..99005d86 100644 --- a/internal/config/repair.go +++ b/internal/config/repair.go @@ -51,7 +51,7 @@ func (c Config) Archives(ctx context.Context, exec boil.ContextExecutor) error { if exec == nil { return fmt.Errorf("config repair archives %w", ErrCE) } - tick := time.Now() + d := time.Now() downloadDir, logger := c.AbsDownload, helper.Logger(ctx) artifacts := []string{} var err error @@ -133,7 +133,7 @@ func (c Config) Archives(ctx context.Context, exec boil.ContextExecutor) error { logger.Errorf("walk directory %s: %s", err, downloadDir) } } - logger.Infof("Completed UUID archive checks in %s", time.Since(tick)) + logger.Infof("Completed UUID archive checks in %.1fs", time.Since(d).Seconds()) return nil } @@ -285,7 +285,7 @@ func (c Config) Assets(ctx context.Context, exec boil.ContextExecutor) error { if exec == nil { return fmt.Errorf("config repair assets %w", ErrCE) } - tick := time.Now() + d := time.Now() logger := helper.Logger(ctx) mods := []qm.QueryMod{} mods = append(mods, qm.Select("uuid")) @@ -338,7 +338,7 @@ func (c Config) Assets(ctx context.Context, exec boil.ContextExecutor) error { for _, count := range counters { sum += count } - logger.Infof("Checked %d files for %d UUIDs in %s", sum, size, time.Since(tick)) + logger.Infof("Checked %d files for %d UUIDs in %.1fs", sum, size, time.Since(d).Seconds()) return nil } @@ -385,6 +385,53 @@ func (c Config) RepairAssets(ctx context.Context, exec boil.ContextExecutor) err if err := c.MagicNumbers(ctx, exec, logger); err != nil { return fmt.Errorf("repair magics %w", err) } + if err := c.TextFiles(ctx, exec, logger); err != nil { + return fmt.Errorf("repair textfiles %w", err) + } + return nil +} + +// TextFiles, on startup check the extra directory for any readme text files that are duplicates of the diz text files. +func (c Config) TextFiles(ctx context.Context, exec boil.ContextExecutor, logger *zap.SugaredLogger) error { + if exec == nil { + return fmt.Errorf("config %w", ErrCE) + } + uuids, err := model.UUID(ctx, exec) + if err != nil { + return fmt.Errorf("config %w", err) + } + dizes := 0 + for _, v := range uuids { + name := filepath.Join(c.AbsExtra, v.UUID.String) + diz := name + ".diz" + txt := name + ".txt" + filed, err := os.Stat(diz) + if err != nil || filed.Size() == 0 { + continue + } + filet, err := os.Stat(txt) + if err != nil || filet.Size() == 0 { + continue + } + if filed.Size() != filet.Size() { + continue + } + dizSI, err := helper.StrongIntegrity(diz) + if err != nil { + continue + } + txtSI, err := helper.StrongIntegrity(txt) + if err != nil { + continue + } + if dizSI == txtSI { + dizes++ + _ = os.Remove(txt) + } + } + if dizes > 0 { + logger.Infof("Found and removed %d readme text files that are duplicates of the diz text files", dizes) + } return nil } diff --git a/model/model.go b/model/model.go index 3036eefc..641529b6 100644 --- a/model/model.go +++ b/model/model.go @@ -2,6 +2,7 @@ package model import ( + "context" "errors" "fmt" "path/filepath" @@ -13,6 +14,7 @@ import ( "github.com/Defacto2/server/model/html3" "github.com/subpop/go-ini" "github.com/volatiletech/sqlboiler/v4/boil" + "github.com/volatiletech/sqlboiler/v4/queries/qm" ) var ( @@ -158,3 +160,11 @@ func JsDosConfig(f *models.File) (string, error) { func invalidExec(exec boil.ContextExecutor) bool { return html3.InvalidExec(exec) } + +// UUID returns a slice of all the UUIDs in the database. +func UUID(ctx context.Context, exec boil.ContextExecutor) (models.FileSlice, error) { + if invalidExec(exec) { + return nil, ErrDB + } + return models.Files(qm.Select("uuid")).All(ctx, exec) +}