fix TextFiles to remove duplicate diz and readme texts used for the s…

…ame uuid record.

flags/flags.go

@@ -10,6 +10,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/Defacto2/server/internal/config"
 	"github.com/carlmjohnson/versioninfo"
@@ -66,7 +67,8 @@ func Fix(c *config.Config) *cli.Command {
 		Usage:       "fix the database and assets",
 		Description: "Fix the database entries and file assets by running scans and checks.",
 		Action: func(_ *cli.Context) error {
-			if err := c.Fixer(); err != nil {
+			d := time.Now()
+			if err := c.Fixer(d); err != nil {
 				return fmt.Errorf("command fix: %w", err)
 			}
 			return nil

internal/config/fixer.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/Defacto2/helper"
 	"github.com/Defacto2/server/internal/command"
@@ -16,7 +17,7 @@ import (
 )
 
 // Fixer is used to fix any known issues with the file assets and the database entries.
-func (c Config) Fixer() error {
+func (c Config) Fixer(d time.Time) error {
 	logger := zaplog.Timestamp().Sugar()
 	db, err := postgres.Open()
 	if err != nil {
@@ -42,6 +43,7 @@ func (c Config) Fixer() error {
 	c.repairer(ctx, db)
 	c.sanityChecks(ctx)
 	SanityTmpDir()
+	logger.Infof("Fixer completed in %.1fs", time.Since(d).Seconds())
 	return nil
 }
 

internal/config/repair.go

@@ -51,7 +51,7 @@ func (c Config) Archives(ctx context.Context, exec boil.ContextExecutor) error {
 	if exec == nil {
 		return fmt.Errorf("config repair archives %w", ErrCE)
 	}
-	tick := time.Now()
+	d := time.Now()
 	downloadDir, logger := c.AbsDownload, helper.Logger(ctx)
 	artifacts := []string{}
 	var err error
@@ -133,7 +133,7 @@ func (c Config) Archives(ctx context.Context, exec boil.ContextExecutor) error {
 			logger.Errorf("walk directory %s: %s", err, downloadDir)
 		}
 	}
-	logger.Infof("Completed UUID archive checks in %s", time.Since(tick))
+	logger.Infof("Completed UUID archive checks in %.1fs", time.Since(d).Seconds())
 	return nil
 }
 
@@ -285,7 +285,7 @@ func (c Config) Assets(ctx context.Context, exec boil.ContextExecutor) error {
 	if exec == nil {
 		return fmt.Errorf("config repair assets %w", ErrCE)
 	}
-	tick := time.Now()
+	d := time.Now()
 	logger := helper.Logger(ctx)
 	mods := []qm.QueryMod{}
 	mods = append(mods, qm.Select("uuid"))
@@ -338,7 +338,7 @@ func (c Config) Assets(ctx context.Context, exec boil.ContextExecutor) error {
 	for _, count := range counters {
 		sum += count
 	}
-	logger.Infof("Checked %d files for %d UUIDs in %s", sum, size, time.Since(tick))
+	logger.Infof("Checked %d files for %d UUIDs in %.1fs", sum, size, time.Since(d).Seconds())
 	return nil
 }
 
@@ -385,6 +385,53 @@ func (c Config) RepairAssets(ctx context.Context, exec boil.ContextExecutor) err
 	if err := c.MagicNumbers(ctx, exec, logger); err != nil {
 		return fmt.Errorf("repair magics %w", err)
 	}
+	if err := c.TextFiles(ctx, exec, logger); err != nil {
+		return fmt.Errorf("repair textfiles %w", err)
+	}
+	return nil
+}
+
+// TextFiles, on startup check the extra directory for any readme text files that are duplicates of the diz text files.
+func (c Config) TextFiles(ctx context.Context, exec boil.ContextExecutor, logger *zap.SugaredLogger) error {
+	if exec == nil {
+		return fmt.Errorf("config %w", ErrCE)
+	}
+	uuids, err := model.UUID(ctx, exec)
+	if err != nil {
+		return fmt.Errorf("config %w", err)
+	}
+	dizes := 0
+	for _, v := range uuids {
+		name := filepath.Join(c.AbsExtra, v.UUID.String)
+		diz := name + ".diz"
+		txt := name + ".txt"
+		filed, err := os.Stat(diz)
+		if err != nil || filed.Size() == 0 {
+			continue
+		}
+		filet, err := os.Stat(txt)
+		if err != nil || filet.Size() == 0 {
+			continue
+		}
+		if filed.Size() != filet.Size() {
+			continue
+		}
+		dizSI, err := helper.StrongIntegrity(diz)
+		if err != nil {
+			continue
+		}
+		txtSI, err := helper.StrongIntegrity(txt)
+		if err != nil {
+			continue
+		}
+		if dizSI == txtSI {
+			dizes++
+			_ = os.Remove(txt)
+		}
+	}
+	if dizes > 0 {
+		logger.Infof("Found and removed %d readme text files that are duplicates of the diz text files", dizes)
+	}
 	return nil
 }
 

model/model.go

@@ -2,6 +2,7 @@
 package model
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"path/filepath"
@@ -13,6 +14,7 @@ import (
 	"github.com/Defacto2/server/model/html3"
 	"github.com/subpop/go-ini"
 	"github.com/volatiletech/sqlboiler/v4/boil"
+	"github.com/volatiletech/sqlboiler/v4/queries/qm"
 )
 
 var (
@@ -158,3 +160,11 @@ func JsDosConfig(f *models.File) (string, error) {
 func invalidExec(exec boil.ContextExecutor) bool {
 	return html3.InvalidExec(exec)
 }
+
+// UUID returns a slice of all the UUIDs in the database.
+func UUID(ctx context.Context, exec boil.ContextExecutor) (models.FileSlice, error) {
+	if invalidExec(exec) {
+		return nil, ErrDB
+	}
+	return models.Files(qm.Select("uuid")).All(ctx, exec)
+}