diff --git a/Taskfile.dist.yaml b/Taskfile.dist.yaml index 1806a068..5e127440 100644 --- a/Taskfile.dist.yaml +++ b/Taskfile.dist.yaml @@ -132,6 +132,12 @@ tasks: desc: "Run the internal web server in development mode with live reload." cmds: - task: serve-linux + serve-fix: + aliases: + - "fix" + desc: "Run the internal web server with the fix flag." + cmds: + - cmd: go run server.go fix serve-linux: internal: true platforms: [linux, freebsd, darwin] diff --git a/cmd/cmd.go b/cmd/cmd.go index 5e780800..b24251b0 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -19,11 +19,12 @@ import ( ) const ( - Title = "Defacto2 web application" // Title of this program. - Domain = "defacto2.net" // Domain of the website. - Program = "defacto2-server" // Program is the command line name of this program. - Author = "Ben Garrett" // Author is the primary programmer of this program. - Email = "contact@defacto2.net" // Email contact for public display. + Title = "Defacto2 web application" // Title of this program. + Domain = "defacto2.net" // Domain of the website. + Program = "defacto2-server" // Program is the command line name of this program. + Author = "Ben Garrett" // Author is the primary programmer of this program. + Email = "contact@defacto2.net" // Email contact for public display. + RecentYear = 2024 // Most recent year of compilation for this program. ) var ErrCmd = errors.New("cannot run command as config is nil") @@ -35,12 +36,11 @@ func App(ver string, c *config.Config) *cli.App { Name: Title, Version: Version(ver), Usage: "serve the Defacto2 web site", - UsageText: "defacto2-server" + - "\ndefacto2-server [command]" + - "\ndefacto2-server [command] --help" + - "\ndefacto2-server [flag]", + UsageText: Program + + "\n" + Program + " [command]" + + "\n" + Program + " [command] --help" + + "\n" + Program + " [flag]", Description: desc(c), - Compiled: versioninfo.LastCommit, Copyright: Copyright(), HelpName: Program, Authors: []*cli.Author{ @@ -50,13 +50,28 @@ func App(ver string, c *config.Config) *cli.App { }, }, Commands: []*cli.Command{ - Config(c), - Address(c), + Config(c), Address(c), Fix(c), }, } return app } +// Fix is the `fix` command help and action. +func Fix(c *config.Config) *cli.Command { + return &cli.Command{ + Name: "fix", + Aliases: []string{"f"}, + Usage: "fix the database and assets", + Description: "Fix the database entries and file assets by running scans and checks.", + Action: func(_ *cli.Context) error { + if err := c.Fixer(); err != nil { + return fmt.Errorf("command fix: %w", err) + } + return nil + }, + } +} + // Address is the `address` command help and action. func Address(c *config.Config) *cli.Command { return &cli.Command{ @@ -120,10 +135,6 @@ func Commit(ver string) string { } else if s != "" { x = append(x, s) } - built := "built at" - if l := LastCommit(); l != "" && !strings.Contains(ver, built) { - x = append(x, built+" "+l) - } if len(x) == 0 || x[0] == "devel" { return "n/a (not a build)" } @@ -135,23 +146,14 @@ func Commit(ver string) string { func Copyright() string { const initYear = 2023 years := strconv.Itoa(initYear) - t := versioninfo.LastCommit - if t.Year() > initYear { - years += "-" + t.Local().Format("06") //nolint:gosmopolitan + if RecentYear > initYear { + const endDigits = RecentYear % 100 + years += "-" + strconv.Itoa(endDigits) } s := fmt.Sprintf("© %s Defacto2 & %s", years, Author) return s } -// LastCommit returns the date of the last repository commit. -func LastCommit() string { - d := versioninfo.LastCommit - if d.IsZero() { - return "" - } - return d.Local().Format("2006 Jan 2 15:04") //nolint:gosmopolitan -} - // OS returns the program operating system. func OS() string { t := cases.Title(language.English) diff --git a/internal/config/check.go b/internal/config/check.go index 607f790f..86d44b19 100644 --- a/internal/config/check.go +++ b/internal/config/check.go @@ -3,12 +3,17 @@ package config // Package file check.go contains the sanity check functions for the configuration values. import ( + "context" + "database/sql" "errors" "fmt" "os" "path/filepath" "github.com/Defacto2/server/internal/helper" + "github.com/Defacto2/server/internal/postgres/models" + "github.com/Defacto2/server/model" + "github.com/volatiletech/sqlboiler/v4/queries/qm" "go.uber.org/zap" ) @@ -205,6 +210,30 @@ func CheckDir(name, desc string) error { return nil } +// RecordCount returns the number of records in the database. +func RecordCount(ctx context.Context, db *sql.DB) int { + if db == nil { + return 0 + } + fs, err := models.Files(qm.Where(model.ClauseNoSoftDel)).Count(ctx, db) + if err != nil { + return 0 + } + return int(fs) +} + +// SanityTmpDir is used to print the temporary directory and its disk usage. +func SanityTmpDir() { + tmpdir := helper.TmpDir() + du, err := helper.DiskUsage(tmpdir) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + hdu := helper.ByteCountFloat(du) + fmt.Fprintf(os.Stdout, "Temporary directory using, %s: %s\n", hdu, tmpdir) +} + // Validate returns an error if the HTTP or TLS port is invalid. func Validate(port uint) error { const disabled = 0 diff --git a/internal/config/config.go b/internal/config/config.go index ab1a0e39..31f1c8aa 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -41,7 +41,11 @@ const ( line = "─" ) -var ErrNoPort = errors.New("the server cannot start without a http or a tls port") +var ( + ErrNoPort = errors.New("the server cannot start without a http or a tls port") + ErrPointer = errors.New("pointer is nil") + ErrVer = errors.New("postgresql version request failed") +) // Configuration is a struct that holds the configuration options. type Configuration struct { @@ -77,7 +81,7 @@ type Config struct { TLSPort uint `env:"D2_TLS_PORT" help:"The port number to be used by the encrypted, HTTPS web server"` Quiet bool `env:"D2_QUIET" help:"Suppress most startup output to the terminal, intended for use with systemd or other process managers"` Compression bool `env:"D2_COMPRESSION" help:"Enable gzip compression of the HTTP/HTTPS responses; you may turn this off when using a reverse proxy"` - ProdMode bool `env:"D2_PROD_MODE" help:"Use the production mode to run checks on startup, log errors to files and recover from panics"` + ProdMode bool `env:"D2_PROD_MODE" help:"Use the production mode to log errors to files and recover from panics"` ReadOnly bool `env:"D2_READ_ONLY" help:"Use the read-only mode to turn off all POST, PUT, and DELETE requests and any related user interface"` NoCrawl bool `env:"D2_NO_CRAWL" help:"Tell search engines to not crawl any of website pages or assets"` LogAll bool `env:"D2_LOG_ALL" help:"Log all HTTP and HTTPS client requests including those with 200 OK responses"` diff --git a/internal/config/fixer.go b/internal/config/fixer.go new file mode 100644 index 00000000..ee34b355 --- /dev/null +++ b/internal/config/fixer.go @@ -0,0 +1,131 @@ +package config + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "strings" + + "github.com/Defacto2/server/internal/command" + "github.com/Defacto2/server/internal/helper" + "github.com/Defacto2/server/internal/postgres" + "github.com/Defacto2/server/internal/zaplog" + "github.com/Defacto2/server/model/fix" +) + +// Fixer is used to fix any known issues with the file assets and the database entries. +func (c Config) Fixer() error { + logger := zaplog.Timestamp().Sugar() + db, err := postgres.Open() + if err != nil { + logger.Errorf("fix could not initialize the database data: %s", err) + } + defer db.Close() + var database postgres.Version + if err := database.Query(db); err != nil { + logger.Errorf("postgres version query: %w", err) + } + fmt.Fprintf(os.Stdout, "\n%+v\n", c) + ctx := context.WithValue(context.Background(), helper.LoggerKey, logger) + count := RecordCount(ctx, db) + const welcome = "Defacto2 web application" + switch { + case count == 0: + logger.Error(welcome + " with no database records") + case MinimumFiles > count: + logger.Warnf(welcome+" with only %d records, expecting at least %d+", count, MinimumFiles) + default: + logger.Infof(welcome+" using %d records", count) + } + c.repairer(ctx, db) + c.sanityChecks(ctx) + SanityTmpDir() + return nil +} + +// repairer is used to fix any known issues with the file assets and the database entries. +// These are skipped if the Production mode environment variable is set to false. +func (c Config) repairer(ctx context.Context, db *sql.DB) { + if db == nil { + panic(fmt.Errorf("%w: repairer", ErrPointer)) + } + logger := helper.Logger(ctx) + if err := c.RepairAssets(ctx, db); err != nil { + logger.Errorf("asset repairs: %s", err) + } + if err := repairDatabase(ctx, db); err != nil { + if errors.Is(err, ErrVer) { + logger.Warnf("A %s, is the database server down?", ErrVer) + } + logger.Errorf("repair database could not initialize the database data: %s", err) + } +} + +// repairDatabase on startup checks the database connection and make any data corrections. +func repairDatabase(ctx context.Context, db *sql.DB) error { + if db == nil { + panic(fmt.Errorf("%w: repair database", ErrPointer)) + } + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("repair database could not begin a transaction: %w", err) + } + if err := fix.Artifacts.Run(ctx, db, tx); err != nil { + defer func() { + if err := tx.Rollback(); err != nil { + logger := helper.Logger(ctx) + logger.Error(err) + } + }() + return fmt.Errorf("repair database could not fix all artifacts: %w", err) + } + return nil +} + +// sanityChecks is used to perform a number of sanity checks on the file assets and database. +// These are skipped if the Production mode environment variable is set.to false. +func (c Config) sanityChecks(ctx context.Context) { + logger := helper.Logger(ctx) + if err := c.Checks(logger); err != nil { + logger.Errorf("sanity checks could not read the environment variable, "+ + "it probably contains an invalid value: %s", err) + } + cmdChecks(ctx) + conn, err := postgres.New() + if err != nil { + logger.Errorf("sanity checks could not initialize the database data: %s", err) + return + } + if err := conn.Validate(logger); err != nil { + panic(fmt.Errorf("sanity check conn validate: %w", err)) + } +} + +// checks is used to confirm the required commands are available. +// These are skipped if readonly is true. +func cmdChecks(ctx context.Context) { + logger := helper.Logger(ctx) + var buf strings.Builder + for i, name := range command.Lookups() { + if err := command.LookCmd(name); err != nil { + buf.WriteString("\n\t\t\tmissing: " + name) + buf.WriteString("\t" + command.Infos()[i]) + } + } + if buf.Len() > 0 { + logger.Warnln("The following commands are required for the server to run in WRITE MODE", + "\n\t\t\tThese need to be installed and accessible on the system path:"+ + "\t\t\t"+buf.String()) + } + if err := command.LookupUnrar(); err != nil { + if errors.Is(err, command.ErrVers) { + logger.Warnf("Found unrar but " + + "could not find unrar by Alexander Roshal, " + + "is unrar-free mistakenly installed?") + return + } + logger.Warnf("lookup unrar check: %s", err) + } +} diff --git a/internal/render/render_test.go b/internal/render/render_test.go index 51a18654..0eea9d6d 100644 --- a/internal/render/render_test.go +++ b/internal/render/render_test.go @@ -1,12 +1,10 @@ package render_test import ( - "encoding/binary" "os" "path/filepath" "strings" "testing" - "unicode/utf16" "github.com/Defacto2/server/internal/helper" "github.com/Defacto2/server/internal/postgres/models" @@ -111,18 +109,6 @@ func TestRead(t *testing.T) { assert.Equal(t, string(b), string(s)) } -func stringToUTF16(s string) []uint16 { - return utf16.Encode([]rune(s)) -} - -func uint16ArrayToByteArray(nums []uint16) []byte { - bytes := make([]byte, len(nums)*2) - for i, num := range nums { - binary.LittleEndian.PutUint16(bytes[i*2:], num) - } - return bytes -} - func TestViewer(t *testing.T) { t.Parallel() var art models.File diff --git a/server.go b/server.go index e42be875..bb548e53 100644 --- a/server.go +++ b/server.go @@ -27,44 +27,32 @@ import ( "github.com/Defacto2/server/cmd" "github.com/Defacto2/server/handler" - "github.com/Defacto2/server/internal/command" "github.com/Defacto2/server/internal/config" "github.com/Defacto2/server/internal/helper" "github.com/Defacto2/server/internal/postgres" - "github.com/Defacto2/server/internal/postgres/models" "github.com/Defacto2/server/internal/zaplog" - "github.com/Defacto2/server/model" - "github.com/Defacto2/server/model/fix" "github.com/caarlos0/env/v11" _ "github.com/lib/pq" - "github.com/volatiletech/sqlboiler/v4/queries/qm" "go.uber.org/zap" ) -//go:embed public/text/defacto2.txt -var brand []byte - -//go:embed public/**/* -var public embed.FS - -//go:embed view/**/* -var view embed.FS - -// version is generated by the GoReleaser ldflags. -var version string - var ( - ErrLog = errors.New("cannot save logs") - ErrPointer = errors.New("pointer is nil") - ErrVer = errors.New("postgresql version request failed") + //go:embed public/text/defacto2.txt + brand []byte + //go:embed public/**/* + public embed.FS + //go:embed view/**/* + view embed.FS + version string // version is generated by the GoReleaser ldflags. ) +var ErrLog = errors.New("cannot save logs") + // Main is the entry point for the application. // By default the web server runs when no arguments are provided, // otherwise, the command-line arguments are parsed and the application exits. func main() { const exit = 0 - // initialize a temporary logger, get and print the environment variable configurations. logger, configs := environmentVars() if exitCode := parseFlags(logger, configs); exitCode >= exit { @@ -83,16 +71,15 @@ func main() { logger.Errorf("main could not initialize the database data: %s", err) } defer db.Close() - ctx := context.WithValue(context.Background(), helper.LoggerKey, logger) var database postgres.Version if err := database.Query(db); err != nil { logger.Errorf("postgres version query: %w", err) } - repairs(ctx, db, configs) - sanityChecks(ctx, configs) - sanityTmpDir() + config.SanityTmpDir() + fmt.Fprintln(w) // start the web server and the sugared logger. + ctx := context.Background() website := newInstance(ctx, db, configs) logger = serverLog(configs, website.RecordCount) router := website.Controller(db, logger) @@ -119,7 +106,7 @@ func main() { } fmt.Fprintf(w, "%s\n", localIPs) }() - // shutdown the web server. + // shutdown the web server after a signal is received. website.ShutdownHTTP(router, logger) } @@ -139,7 +126,6 @@ func environmentVars() (*zap.SugaredLogger, config.Config) { logger.Fatalf("could not parse the environment variable, it probably contains an invalid value: %s", err) } configs.Override() - if i := configs.MaxProcs; i > 0 { runtime.GOMAXPROCS(int(i)) } @@ -159,7 +145,7 @@ func newInstance(ctx context.Context, db *sql.DB, configs config.Config) handler c.Version = cmd.Commit("") } if ctx != nil && db != nil { - c.RecordCount = recordCount(ctx, db) + c.RecordCount = config.RecordCount(ctx, db) } return c } @@ -183,86 +169,6 @@ func parseFlags(logger *zap.SugaredLogger, configs config.Config) int { return -1 } -// sanityChecks is used to perform a number of sanity checks on the file assets and database. -// These are skipped if the Production mode environment variable is set.to false. -func sanityChecks(ctx context.Context, configs config.Config) { - if !configs.ProdMode { - return - } - logger := helper.Logger(ctx) - if err := configs.Checks(logger); err != nil { - logger.Errorf("sanity checks could not read the environment variable, "+ - "it probably contains an invalid value: %s", err) - } - checks(logger, configs.ReadOnly) - conn, err := postgres.New() - if err != nil { - logger.Errorf("sanity checks could not initialize the database data: %s", err) - return - } - _ = conn.Validate(logger) -} - -// sanityTmpDir is used to print the temporary directory and its disk usage. -func sanityTmpDir() { - tmpdir := helper.TmpDir() - du, err := helper.DiskUsage(tmpdir) - if err != nil { - fmt.Fprintln(os.Stderr, err) - return - } - hdu := helper.ByteCountFloat(du) - fmt.Fprintf(os.Stdout, "Temporary directory using, %s: %s\n\n", hdu, tmpdir) -} - -// checks is used to confirm the required commands are available. -// These are skipped if readonly is true. -func checks(logger *zap.SugaredLogger, readonly bool) { - if logger == nil || readonly { - return - } - var buf strings.Builder - for i, name := range command.Lookups() { - if err := command.LookCmd(name); err != nil { - buf.WriteString("\n\t\t\tmissing: " + name) - buf.WriteString("\t" + command.Infos()[i]) - } - } - if buf.Len() > 0 { - logger.Warnln("The following commands are required for the server to run in WRITE MODE", - "\n\t\t\tThese need to be installed and accessible on the system path:"+ - "\t\t\t"+buf.String()) - } - if err := command.LookupUnrar(); err != nil { - if errors.Is(err, command.ErrVers) { - logger.Warnf("Found unrar but " + - "could not find unrar by Alexander Roshal, " + - "is unrar-free mistakenly installed?") - return - } - logger.Warnf("lookup unrar check: %s", err) - } -} - -// repairs is used to fix any known issues with the file assets and the database entries. -// These are skipped if the Production mode environment variable is set to false. -func repairs(ctx context.Context, db *sql.DB, configs config.Config) { - if !configs.ProdMode || db == nil { - return - } - logger := helper.Logger(ctx) - if err := configs.RepairAssets(ctx, db); err != nil { - logger.Errorf("asset repairs: %s", err) - } - err := repairDatabase(ctx, db) - if err != nil { - if errors.Is(err, ErrVer) { - logger.Warnf("A %s, is the database server down?", ErrVer) - } - logger.Errorf("repair database could not initialize the database data: %s", err) - } -} - // serverLog is used to setup the logger for the server and print the startup message. func serverLog(configs config.Config, count int) *zap.SugaredLogger { logger := zaplog.Timestamp().Sugar() @@ -283,37 +189,3 @@ func serverLog(configs config.Config, count int) *zap.SugaredLogger { } return logger } - -// repairDatabase on startup checks the database connection and make any data corrections. -func repairDatabase(ctx context.Context, db *sql.DB) error { - if db == nil { - return fmt.Errorf("%w: %s", ErrPointer, - "the repair database is missing a required parameter") - } - logger := helper.Logger(ctx) - tx, err := db.Begin() - if err != nil { - return fmt.Errorf("repair database could not begin a transaction: %w", err) - } - if err := fix.Artifacts.Run(ctx, db, tx); err != nil { - defer func() { - if err := tx.Rollback(); err != nil { - logger.Error(err) - } - }() - return fmt.Errorf("repair database could not fix all artifacts: %w", err) - } - return nil -} - -// recordCount returns the number of records in the database. -func recordCount(ctx context.Context, db *sql.DB) int { - if db == nil { - return 0 - } - fs, err := models.Files(qm.Where(model.ClauseNoSoftDel)).Count(ctx, db) - if err != nil { - return 0 - } - return int(fs) -}