diff --git a/handler/app/web.go b/handler/app/web.go index b64d5910..d3eebfd3 100644 --- a/handler/app/web.go +++ b/handler/app/web.go @@ -9,6 +9,7 @@ import ( "html/template" "path/filepath" "reflect" + "strconv" "strings" "time" @@ -16,6 +17,7 @@ import ( "github.com/Defacto2/releaser/initialism" "github.com/Defacto2/releaser/name" "github.com/Defacto2/server/internal/config" + "github.com/Defacto2/server/internal/demozoo" "github.com/Defacto2/server/internal/helper" "github.com/volatiletech/null/v8" "go.uber.org/zap" @@ -222,6 +224,9 @@ func (web Web) TemplateClosures() template.FuncMap { "bootstrap5JS": func() string { return hrefs[Bootstrap5JS] }, + "demozooSanity": func() string { + return strconv.Itoa(demozoo.Sanity) + }, "editArchive": func() string { return hrefs[EditArchive] }, diff --git a/handler/htmx/demozoo.go b/handler/htmx/demozoo.go index 5d223696..62dc8ccc 100644 --- a/handler/htmx/demozoo.go +++ b/handler/htmx/demozoo.go @@ -2,31 +2,19 @@ package htmx import ( "context" - "errors" "fmt" "net/http" - "os" - "path/filepath" "strconv" "strings" - "github.com/Defacto2/server/internal/archive" "github.com/Defacto2/server/internal/demozoo" "github.com/Defacto2/server/internal/helper" "github.com/Defacto2/server/internal/postgres" "github.com/Defacto2/server/model" - "github.com/gabriel-vasile/mimetype" "github.com/labstack/echo/v4" - "github.com/volatiletech/null/v8" - "github.com/volatiletech/sqlboiler/v4/boil" "go.uber.org/zap" ) -var ( - ErrDB = errors.New("database connection is nil") - ErrExist = errors.New("file already exists") -) - // DemozooProd fetches the multiple download_links values from the // Demozoo production API and attempts to download and save one of the // linked files. If multiple links are found, the first link is used as @@ -35,9 +23,10 @@ var ( // Both the Demozoo production ID param and the Defacto2 UUID query // param values are required as params to fetch the production data and // to save the file to the correct filename. -func DemozooProd(logr *zap.SugaredLogger, c echo.Context, downloadDir string) error { +func DemozooProd(logr *zap.SugaredLogger, c echo.Context) error { if logr == nil { - return c.String(http.StatusInternalServerError, "Error, demozoo prod logger is nil") + return c.String(http.StatusInternalServerError, + "error, demozoo prod logger is nil") } sid := c.FormValue("demozoo-submission") id, err := strconv.Atoi(sid) @@ -53,42 +42,65 @@ func DemozooProd(logr *zap.SugaredLogger, c echo.Context, downloadDir string) er defer db.Close() ctx := context.Background() - key, err := model.FindDemozooFile(ctx, db, int64(id)) + deleted, key, err := model.FindDemozooFile(ctx, db, int64(id)) if err != nil { - return c.String(http.StatusServiceUnavailable, "Error, the database query failed") + return c.String(http.StatusServiceUnavailable, + "error, the database query failed") } - if key != 0 { - // ID to test: 198232 + if key != 0 && !deleted { html := fmt.Sprintf("This Demozoo production is already in use.", helper.ObfuscateID(key)) return c.HTML(http.StatusOK, html) } + if key != 0 && deleted { + return c.HTML(http.StatusOK, "This Demozoo production is already in use.") + } - info, err := DemozooValid(c, id) + prod, err := DemozooValid(c, id) if err != nil { return err } - if info == "" { + if prod.ID < 1 { return nil } - // ID to test: 66654 - html := `
` - html += fmt.Sprintf(``, id) + + info := []string{prod.Title} + if len(prod.Authors) > 0 { + info = append(info, "by") + for _, a := range prod.Authors { + info = append(info, a.Name) + } + } + if prod.ReleaseDate != "" { + info = append(info, "on", prod.ReleaseDate) + } + if prod.Platforms != nil { + info = append(info, "for") + for _, p := range prod.Platforms { + info = append(info, p.Name) + } + } + html := `
` + html += fmt.Sprintf(``, id, id) html += `
` - html += fmt.Sprintf(`

%s

`, info) + html += fmt.Sprintf(`

%s

`, strings.Join(info, " ")) return c.HTML(http.StatusOK, html) } // DemozooValid fetches the first usable download link from the Demozoo API. // The production ID is validated and the production is checked to see if it -// is suitable for Defacto2. If suitable, the production title and -// author groups are returned. -func DemozooValid(c echo.Context, id int) (string, error) { +// is suitable for Defacto2. If the production is not suitable, an empty +// production is returned with a htmx message. +func DemozooValid(c echo.Context, id int) (demozoo.Production, error) { if id < 1 { - return "", c.String(http.StatusNotAcceptable, fmt.Sprintf("invalid id: %d", id)) + return demozoo.Production{}, + c.String(http.StatusNotAcceptable, fmt.Sprintf("invalid id: %d", id)) } + var prod demozoo.Production if code, err := prod.Get(id); err != nil { - return "", c.String(code, err.Error()) + return demozoo.Production{}, c.String(code, err.Error()) } plat, sect := prod.SuperType() if plat == -1 || sect == -1 { @@ -99,174 +111,69 @@ func DemozooValid(c echo.Context, id int) (string, error) { for _, t := range prod.Types { s = append(s, t.Name) } - return "", c.HTML(http.StatusOK, + return demozoo.Production{}, c.HTML(http.StatusOK, fmt.Sprintf("Production %d is probably not suitable for Defacto2.
Types: %s", id, strings.Join(s, " - "))) } - var ok string + var valid string for _, link := range prod.DownloadLinks { if link.URL == "" { continue } - ok = link.URL + valid = link.URL break } - if ok == "" { - return "", c.String(http.StatusOK, "This Demozoo production has no suitable download links.") + if valid == "" { + return demozoo.Production{}, + c.String(http.StatusOK, "This Demozoo production has no suitable download links.") } - s := []string{fmt.Sprintf("%q", prod.Title)} - for _, a := range prod.Authors { - if a.Releaser.IsGroup { - s = append(s, a.Releaser.Name) - } - } - return strings.Join(s, " "), nil -} - -// Production is the response from the task of GetDemozooFile. -// -//nolint:tagliatelle -type Production struct { - UUID string `json:"uuid"` // UUID is the file production UUID. - Filename string `json:"filename"` // Filename is the file name of the download. - FileType string `json:"file_type"` // Type is the file type. - FileHash string `json:"file_hash"` // Hash is the file integrity hash. - Content string `json:"content"` // Content is the file archive content. - Readme string `json:"readme"` // Readme is the file readme, text or NFO file. - LinkURL string `json:"link_url"` // LinkURL is the download file link used to fetch the file. - LinkClass string `json:"link_class"` // LinkClass is the download link class provided by Demozoo. - Error string `json:"error"` // Error is the error message if the download or record update failed. - Github string `json:"github_repo"` - YouTube string `json:"youtube_video"` - ID int `json:"id"` // ID is the Demozoo production ID. - FileSize int `json:"file_size"` // Size is the file size in bytes. - Pouet int `json:"pouet_prod"` - Success bool `json:"success"` // Success is the success status of the download and record update. + return prod, nil } -func (got *Production) Download(c echo.Context, downloadDir string) error { - var rec demozoo.Production - if _, err := rec.Get(got.ID); err != nil { - got.Error = fmt.Errorf("could not get record %d from demozoo api: %w", got.ID, err).Error() - return c.JSON(http.StatusInternalServerError, got) - } - for _, link := range rec.DownloadLinks { - if link.URL == "" { - continue - } - df, err := helper.DownloadFile(link.URL) - if err != nil || df.Path == "" { - // continue, to attempt the next download link - continue - } - base := filepath.Base(link.URL) - dst := filepath.Join(downloadDir, got.UUID) - got.Filename = base - got.LinkClass = link.LinkClass - got.LinkURL = link.URL - if err := helper.RenameFileOW(df.Path, dst); err != nil { - // if the rename file fails, check if the uuid file asset already exists - // and if it is the same as the downloaded file, if not then return an error. - sameFiles, err := helper.FileMatch(df.Path, dst) - if err != nil { - got.Error = fmt.Errorf("could not rename file, %s: %w", dst, err).Error() - return c.JSON(http.StatusInternalServerError, got) - } - if !sameFiles { - got.Error = fmt.Errorf("%w, will not overwrite, %s", ErrExist, dst).Error() - return c.JSON(http.StatusConflict, got) - } - } - // get the file size - size, err := strconv.Atoi(df.ContentLength) - if err == nil { - got.FileSize = size - } - // get the file type - if df.ContentType != "" { - got.FileType = df.ContentType - } - got.Filename = base - got.LinkURL = link.URL - got.LinkClass = link.LinkClass - got.Success = true - got.Error = "" - // obtain data from the external links - got.Github = rec.GithubRepo() - got.Pouet = rec.PouetProd() - got.YouTube = rec.YouTubeVideo() - return got.Stat(c, downloadDir) +// DemozooSubmit is the handler for the /demozoo/production/submit route. +// This will attempt to insert a new file record into the database using +// the Demozoo production ID. If the Demozoo production ID is already in +// use, an error message is returned. +func DemozooSubmit(logr *zap.SugaredLogger, c echo.Context) error { + if logr == nil { + return c.String(http.StatusInternalServerError, + "error, demozoo submit logger is nil") } - got.Error = "no usable download links found, they returned 404 or were empty" - return c.JSON(http.StatusNotModified, got) -} -func (got *Production) Stat(c echo.Context, downloadDir string) error { - path := filepath.Join(downloadDir, got.UUID) - // get the file size if not already set - if got.FileSize == 0 { - stat, err := os.Stat(path) - if err != nil { - got.Error = fmt.Errorf("could not stat file, %s: %w", path, err).Error() - return c.JSON(http.StatusInternalServerError, got) - } - got.FileSize = int(stat.Size()) - } - // get the file integrity hash - strong, err := helper.StrongIntegrity(path) + sid := c.Param("id") + id, err := strconv.ParseUint(sid, 10, 64) if err != nil { - got.Error = fmt.Errorf("could not get strong integrity hash, %s: %w", path, err).Error() - return c.JSON(http.StatusInternalServerError, got) - } - got.FileHash = strong - // get the file type if not already set - if got.FileType == "" { - m, err := mimetype.DetectFile(path) - if err != nil { - return fmt.Errorf("content filemime failure on %q: %w", path, err) - } - got.FileType = m.String() + return c.String(http.StatusNotAcceptable, + "The Demozoo production ID must be a numeric value, "+sid) } - return got.ArchiveContent(c, path) -} - -func (got *Production) ArchiveContent(c echo.Context, path string) error { - files, err := archive.List(path, got.Filename) - if err != nil { - return c.JSON(http.StatusOK, got) + if id < 1 || id > demozoo.Sanity { + return c.String(http.StatusNotAcceptable, + "The Demozoo production ID is invalid, "+sid) } - got.Readme = archive.Readme(got.Filename, files...) - got.Content = strings.Join(files, "\n") - return got.Update(c) -} -func (got Production) Update(c echo.Context) error { - uid := got.UUID db, err := postgres.ConnectDB() if err != nil { return ErrDB } defer db.Close() ctx := context.Background() - f, err := model.OneByUUID(ctx, db, true, uid) - if err != nil { - return err + + if exist, err := model.ExistDemozooFile(ctx, db, int64(id)); err != nil { + return c.String(http.StatusServiceUnavailable, + "error, the database query failed") + } else if exist { + return c.String(http.StatusForbidden, + "error, the demozoo key is already in use") } - f.Filename = null.StringFrom(got.Filename) - f.Filesize = int64(got.FileSize) - f.FileMagicType = null.StringFrom(got.FileType) - f.FileIntegrityStrong = null.StringFrom(got.FileHash) - f.FileZipContent = null.StringFrom(got.Content) - rm := strings.TrimSpace(got.Readme) - f.RetrotxtReadme = null.StringFrom(rm) - gt := strings.TrimSpace(got.Github) - f.WebIDGithub = null.StringFrom(gt) - f.WebIDPouet = null.Int64From(int64(got.Pouet)) - yt := strings.TrimSpace(got.YouTube) - f.WebIDYoutube = null.StringFrom(yt) - if _, err = f.Update(ctx, db, boil.Infer()); err != nil { - return err + + key, err := model.InsertDemozooFile(ctx, db, int64(id)) + if err != nil || key == 0 { + logr.Error(err, id) + return c.String(http.StatusServiceUnavailable, + "error, the database insert failed") } - return c.JSON(http.StatusOK, got) + + html := fmt.Sprintf("Thanks for the submission of Demozoo production: %d", id) + return c.HTML(http.StatusOK, html) } diff --git a/handler/htmx/htmx.go b/handler/htmx/htmx.go index a14f4cb7..d337b4fa 100644 --- a/handler/htmx/htmx.go +++ b/handler/htmx/htmx.go @@ -2,31 +2,37 @@ package htmx import ( - "context" "embed" + "errors" "html/template" - "net/http" "strings" "github.com/Defacto2/releaser" "github.com/Defacto2/releaser/initialism" "github.com/Defacto2/releaser/name" "github.com/Defacto2/server/handler/app" - "github.com/Defacto2/server/internal/helper" - "github.com/Defacto2/server/internal/postgres" - "github.com/Defacto2/server/model" "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" "go.uber.org/zap" ) +var ( + ErrDB = errors.New("database connection is nil") + ErrExist = errors.New("file already exists") +) + // Routes for the /htmx sub-route group that returns HTML fragments // using the htmx library for AJAX responses. -func Routes(logr *zap.SugaredLogger, e *echo.Echo, dlDir string) *echo.Echo { - e.POST("/search/releaser", func(x echo.Context) error { - return SearchReleaser(logr, x) +func Routes(logr *zap.SugaredLogger, e *echo.Echo, dirDownload string) *echo.Echo { + submit := e.Group("/", middleware.RateLimiter(middleware.NewRateLimiterMemoryStore(2))) + submit.POST("/demozoo/production", func(x echo.Context) error { + return DemozooProd(logr, x) }) - e.POST("/demozoo/production", func(x echo.Context) error { - return DemozooProd(logr, x, dlDir) // dir.Download + submit.POST("/demozoo/production/submit/:id", func(x echo.Context) error { + return DemozooSubmit(logr, x) + }) + submit.POST("/search/releaser", func(x echo.Context) error { + return SearchReleaser(logr, x) }) return e } @@ -37,54 +43,6 @@ func GlobTo(name string) string { return strings.Join([]string{"view", "htmx", name}, "/") } -// SearchReleaser is a handler for the /search/releaser route. -func SearchReleaser(logr *zap.SugaredLogger, c echo.Context) error { - const maxResults = 14 - ctx := context.Background() - db, err := postgres.ConnectDB() - if err != nil { - logr.Error(err) - return c.String(http.StatusServiceUnavailable, - "cannot connect to the database") - } - defer db.Close() - - input := c.FormValue("releaser-search") - slug := helper.Slug(helper.TrimRoundBraket(input)) - if slug == "" { - return c.HTML(http.StatusOK, "") - } - - lookup := []string{} - for key, values := range initialism.Initialisms() { - for _, value := range values { - if strings.Contains(strings.ToLower(value), strings.ToLower(slug)) { - lookup = append(lookup, string(key)) - } - } - } - lookup = append(lookup, slug) - var r model.Releasers - if err := r.Similar(ctx, db, maxResults, lookup...); err != nil { - logr.Error(err) - return c.String(http.StatusServiceUnavailable, - "the search query failed") - } - if len(r) == 0 { - return c.HTML(http.StatusOK, "No releasers found.") - } - err = c.Render(http.StatusOK, "releasers", map[string]interface{}{ - "maximum": maxResults, - "name": slug, - "result": r, - }) - if err != nil { - return c.String(http.StatusInternalServerError, - "cannot render the htmx template") - } - return nil -} - func releasers(fs embed.FS) *template.Template { return template.Must(template.New("").Funcs(TemplateFuncMap()).ParseFS(fs, GlobTo("layout.tmpl"), GlobTo("releasers.tmpl"))) diff --git a/handler/htmx/search.go b/handler/htmx/search.go new file mode 100644 index 00000000..31d5bb46 --- /dev/null +++ b/handler/htmx/search.go @@ -0,0 +1,62 @@ +package htmx + +import ( + "context" + "net/http" + "strings" + + "github.com/Defacto2/releaser/initialism" + "github.com/Defacto2/server/internal/helper" + "github.com/Defacto2/server/internal/postgres" + "github.com/Defacto2/server/model" + "github.com/labstack/echo/v4" + "go.uber.org/zap" +) + +// SearchReleaser is a handler for the /search/releaser route. +func SearchReleaser(logr *zap.SugaredLogger, c echo.Context) error { + const maxResults = 14 + ctx := context.Background() + db, err := postgres.ConnectDB() + if err != nil { + logr.Error(err) + return c.String(http.StatusServiceUnavailable, + "cannot connect to the database") + } + defer db.Close() + + input := c.FormValue("releaser-search") + slug := helper.Slug(helper.TrimRoundBraket(input)) + if slug == "" { + return c.HTML(http.StatusOK, "") + } + + lookup := []string{} + for key, values := range initialism.Initialisms() { + for _, value := range values { + if strings.Contains(strings.ToLower(value), strings.ToLower(slug)) { + lookup = append(lookup, string(key)) + } + } + } + lookup = append(lookup, slug) + var r model.Releasers + if err := r.Similar(ctx, db, maxResults, lookup...); err != nil { + logr.Error(err) + return c.String(http.StatusServiceUnavailable, + "the search query failed") + } + if len(r) == 0 { + return c.HTML(http.StatusOK, "No releasers found.") + } + err = c.Render(http.StatusOK, "releasers", map[string]interface{}{ + "maximum": maxResults, + "name": slug, + "result": r, + }) + if err != nil { + return c.String(http.StatusInternalServerError, + "cannot render the htmx template") + } + return nil +} diff --git a/internal/demozoo/demozoo.go b/internal/demozoo/demozoo.go index 32eac32e..65e7f0a6 100644 --- a/internal/demozoo/demozoo.go +++ b/internal/demozoo/demozoo.go @@ -28,6 +28,7 @@ import ( const ( ProdURL = "https://demozoo.org/api/v1/productions/" // ProdURL is the base URL for the Demozoo production API. Timeout = 10 * time.Second // HTTP client timeout, Demozoo replies can be slow. + Sanity = 450000 // Sanity is to check the maximum permitted production ID. ) // Production is a Demozoo production record. @@ -42,6 +43,7 @@ type Production struct { Supertype string `json:"supertype"` // Authors Authors []struct { + Name string `json:"name"` Releaser struct { Name string `json:"name"` IsGroup bool `json:"is_group"` diff --git a/model/file.go b/model/file.go index 213ee157..d2823d4f 100644 --- a/model/file.go +++ b/model/file.go @@ -5,14 +5,27 @@ import ( "database/sql" "errors" "fmt" + "time" + "github.com/Defacto2/server/internal/demozoo" "github.com/Defacto2/server/internal/helper" "github.com/Defacto2/server/internal/postgres" "github.com/Defacto2/server/internal/postgres/models" + "github.com/google/uuid" "github.com/volatiletech/null/v8" + "github.com/volatiletech/sqlboiler/v4/boil" "github.com/volatiletech/sqlboiler/v4/queries/qm" ) +// FindFile returns true if the file record exists in the database. +// This function will also return true for records that have been marked as deleted. +func ExistFile(ctx context.Context, db *sql.DB, id int64) (bool, error) { + if db == nil { + return false, ErrDB + } + return models.Files(models.FileWhere.ID.EQ(id), qm.WithDeleted()).Exists(ctx, db) +} + // FindFile retrieves a single file record from the database using the record key. // This function will also return records that have been marked as deleted. func FindFile(ctx context.Context, db *sql.DB, id int64) (*models.File, error) { @@ -22,23 +35,59 @@ func FindFile(ctx context.Context, db *sql.DB, id int64) (*models.File, error) { return models.Files(models.FileWhere.ID.EQ(id), qm.WithDeleted()).One(ctx, db) } +// ExistDemozooFile returns true if the file record exists in the database using a Demozoo production ID. +// This function will also return true for records that have been marked as deleted. +func ExistDemozooFile(ctx context.Context, db *sql.DB, id int64) (bool, error) { + if db == nil { + return false, ErrDB + } + return models.Files(models.FileWhere.WebIDDemozoo.EQ(null.Int64From(id)), qm.WithDeleted()).Exists(ctx, db) +} + // FindDemozooFile retrieves the ID or key of a single file record from the database using a Demozoo production ID. -// This function will also return records that have been marked as deleted. -// If the record is not found then the function will return 0 but without an error. -func FindDemozooFile(ctx context.Context, db *sql.DB, id int64) (int64, error) { +// This function will also return records that have been marked as deleted and flag those with the boolean. +// If the record is not found then the function will return an ID of 0 but without an error. +func FindDemozooFile(ctx context.Context, db *sql.DB, id int64) (bool, int64, error) { if db == nil { - return 0, ErrDB + return false, 0, ErrDB } f, err := models.Files( - qm.Select("id"), + qm.Select("id", "deletedat"), models.FileWhere.WebIDDemozoo.EQ(null.Int64From(id)), qm.WithDeleted()).One(ctx, db) if errors.Is(err, sql.ErrNoRows) { - return 0, nil + return false, 0, nil } + if err != nil { + return false, 0, err + } + deleted := !f.Deletedat.IsZero() + return deleted, f.ID, nil +} + +// InsertDemozooFile inserts a new file record into the database using a Demozoo production ID. +// This will not check if the Demozoo production ID already exists in the database. +// When successful the function will return the new record ID. +func InsertDemozooFile(ctx context.Context, db *sql.DB, id int64) (int64, error) { + if db == nil { + return 0, ErrDB + } + if id < startID || id > demozoo.Sanity { + return 0, fmt.Errorf("%w: %d", ErrID, id) + } + uid, err := uuid.NewV7() if err != nil { return 0, err } + now := time.Now() + f := models.File{ + UUID: null.StringFrom(uid.String()), + WebIDDemozoo: null.Int64From(id), + Deletedat: null.TimeFromPtr(&now), + } + if err = f.Insert(ctx, db, boil.Infer()); err != nil { + return 0, err + } return f.ID, nil } @@ -75,7 +124,7 @@ func record(deleted bool, key int) (*models.File, error) { defer db.Close() art, err := One(ctx, db, deleted, key) if err != nil { - return nil, ErrDB + return nil, fmt.Errorf("%w, %w: %d", ErrID, err, key) } return art, nil } @@ -95,7 +144,7 @@ func recordObf(deleted bool, key string) (*models.File, error) { defer db.Close() art, err := One(ctx, db, deleted, id) if err != nil { - return nil, ErrDB + return nil, fmt.Errorf("%w, %w: %s", ErrID, err, key) } if art.ID != int64(id) { return nil, fmt.Errorf("%w: %d ~ %s", ErrID, id, key) diff --git a/view/app/searchHtmx.tmpl b/view/app/searchHtmx.tmpl index 25b3b602..afbc90c8 100644 --- a/view/app/searchHtmx.tmpl +++ b/view/app/searchHtmx.tmpl @@ -31,12 +31,11 @@
- {{- end }} \ No newline at end of file diff --git a/view/app/uploaderHtmx.tmpl b/view/app/uploaderHtmx.tmpl index 12d18718..a2e858ec 100644 --- a/view/app/uploaderHtmx.tmpl +++ b/view/app/uploaderHtmx.tmpl @@ -1,12 +1,11 @@ {{- /* uploaderHtmx.tmpl */}} {{- define "uploaderX" -}} - {{- /* Submit a Demozoo production using htmx */}} - + - {{- end }} \ No newline at end of file