From 401d3cfc820f0d92afed3dc8d9d49155df8c02bf Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Tue, 1 Aug 2023 18:55:36 +0300 Subject: [PATCH] feat: add repology exporter (#498) Co-authored-by: Elsie --- server/Makefile | 1 + server/bin/webserver/main.go | 22 +-- server/config/build/vars.go | 1 + server/config/vars.go | 1 + server/go.sum | 6 - server/model/connection.go | 6 +- server/model/repology_project.go | 11 ++ server/model/repology_project_provider.go | 47 ++++++ server/repology/dto.go | 28 ---- server/repology/exporter.go | 5 + server/repology/fetch.go | 182 ++++++---------------- server/repology/internal/api.go | 40 +++++ server/repology/internal/api_types.go | 18 +++ server/repology/internal/exporter.go | 128 +++++++++++++++ server/repology/lib.go | 4 +- server/repology/retry.go | 48 ------ server/repology/scheduler.go | 25 +++ server/repology/sync.go | 21 ++- server/types/pac/parser/parse.go | 25 +-- server/types/pac/parser/scheduler.go | 2 +- 20 files changed, 372 insertions(+), 249 deletions(-) create mode 100644 server/model/repology_project.go create mode 100644 server/model/repology_project_provider.go delete mode 100644 server/repology/dto.go create mode 100644 server/repology/exporter.go create mode 100644 server/repology/internal/api.go create mode 100644 server/repology/internal/api_types.go create mode 100644 server/repology/internal/exporter.go delete mode 100644 server/repology/retry.go create mode 100644 server/repology/scheduler.go diff --git a/server/Makefile b/server/Makefile index 0312231c..29d232e9 100644 --- a/server/Makefile +++ b/server/Makefile @@ -28,6 +28,7 @@ run: PACSTALL_MATOMO_ENABLED="false" \ PACSTALL_REPOLOGY_ENABLED="false" \ go run bin/webserver/main.go + dist/webserver: $(shell find . -not \( -path ./tmp -prune \) -not \( -path ./dist -prune \) -type f) CGO_ENABLED=0 go build -o dist/webserver -ldflags "${LDFLAGS}" bin/webserver/main.go clean: diff --git a/server/bin/webserver/main.go b/server/bin/webserver/main.go index ab9c237c..e1b6b2df 100644 --- a/server/bin/webserver/main.go +++ b/server/bin/webserver/main.go @@ -7,6 +7,7 @@ import ( "github.com/fatih/color" "pacstall.dev/webserver/config" "pacstall.dev/webserver/log" + "pacstall.dev/webserver/repology" "pacstall.dev/webserver/server" ps_api "pacstall.dev/webserver/server/api/pacscripts" repology_api "pacstall.dev/webserver/server/api/repology" @@ -54,8 +55,6 @@ func main() { } startedAt := time.Now() - port := config.Port - refreshTimer := config.UpdateInterval printLogo() @@ -66,19 +65,20 @@ func main() { server.OnServerOnline(func() { log.NotifyCustom("🚀 Startup 🧑‍🚀", "Successfully started up.") - log.Info("Server is now online on port %v.\n", port) + log.Info("Server is now online on port %v.\n", config.Port) log.Info("Booted in %v\n", color.GreenString("%v", time.Since(startedAt))) - log.Info("Attempting to parse existing pacscripts") - err := parser.ParseAll() - if err != nil { - log.Error("Failed to parse pacscripts: %v", err) - } + parser.ScheduleRefresh(config.UpdateInterval) + log.Info("Scheduled pacscripts to auto-refresh every %v", config.UpdateInterval) - parser.ScheduleRefresh(refreshTimer) - log.Info("Scheduled pacscripts to auto-refresh every %v", refreshTimer) + if config.Repology.Enabled { + repology.ScheduleRefresh(config.RepologyUpdateInterval) + log.Info("Scheduled repology to auto-refresh every %v", config.RepologyUpdateInterval) + } else { + log.Warn("Repology is disabled. Pacstall will not be able to fetch package data from Repology.") + } }) - server.Listen(port) + server.Listen(config.Port) } diff --git a/server/config/build/vars.go b/server/config/build/vars.go index d9e63003..1b1fc378 100644 --- a/server/config/build/vars.go +++ b/server/config/build/vars.go @@ -3,6 +3,7 @@ package build var Production = "false" var UpdateInterval = "900" +var RepologyUpdateInterval = "43200" var TempDir = "./tmp" var MaxOpenFiles = "100" var GitURL = "https://github.com/pacstall/pacstall-programs.git" diff --git a/server/config/vars.go b/server/config/vars.go index 428919f4..d41d7f64 100644 --- a/server/config/vars.go +++ b/server/config/vars.go @@ -9,6 +9,7 @@ import ( var Production = toBool(build.Production) var UpdateInterval = time.Duration(toInt(build.UpdateInterval)) * time.Second +var RepologyUpdateInterval = time.Duration(toInt(build.RepologyUpdateInterval)) * time.Second var TempDir = build.TempDir var MaxOpenFiles = toInt(build.MaxOpenFiles) var GitURL = build.GitURL diff --git a/server/go.sum b/server/go.sum index 056f7816..be446e8d 100644 --- a/server/go.sum +++ b/server/go.sum @@ -48,14 +48,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.5.0 h1:6hSAT5QcyIaty0jfnff0z0CLDjyRgZ8mlMHLqSt7uXM= -gorm.io/driver/mysql v1.5.0/go.mod h1:FFla/fJuCvyTi7rJQd27qlNX2v3L6deTR1GgTjSOLPo= gorm.io/driver/mysql v1.5.1 h1:WUEH5VF9obL/lTtzjmML/5e6VfFR/788coz2uaVCAZw= gorm.io/driver/mysql v1.5.1/go.mod h1:Jo3Xu7mMhCyj8dlrb3WoCaRd1FhsVh+yMXb1jUInf5o= -gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= -gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU= -gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= -gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64= gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= diff --git a/server/model/connection.go b/server/model/connection.go index fa60edf0..c460120c 100644 --- a/server/model/connection.go +++ b/server/model/connection.go @@ -5,6 +5,7 @@ import ( "gorm.io/driver/mysql" "gorm.io/gorm" + "gorm.io/gorm/logger" "pacstall.dev/webserver/config" ) @@ -22,7 +23,10 @@ func Instance() *gorm.DB { return database } - db, err := gorm.Open(mysql.Open(connectionString), &gorm.Config{}) + db, err := gorm.Open(mysql.Open(connectionString), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { panic(fmt.Sprintf("failed to connect database: %v", err)) } diff --git a/server/model/repology_project.go b/server/model/repology_project.go new file mode 100644 index 00000000..3b9b9ff6 --- /dev/null +++ b/server/model/repology_project.go @@ -0,0 +1,11 @@ +package model + +type RepologyProject struct { + Name string `gorm:"primaryKey"` +} + +var RepologyProjectColumns = struct { + Name string +}{ + Name: "name", +} diff --git a/server/model/repology_project_provider.go b/server/model/repology_project_provider.go new file mode 100644 index 00000000..7e410938 --- /dev/null +++ b/server/model/repology_project_provider.go @@ -0,0 +1,47 @@ +package model + +const RepologyProjectProviderTableName = "repology_project_providers" + +type RepologyProjectProvider struct { + ID uint `gorm:"primarykey"` + ProjectName string `gorm:"index:"` + Project RepologyProject `gorm:"foreignKey:Name"` + Repository string + SubRepository *string `gorm:"default:null"` + SourceName *string `gorm:"index:,default:null"` + VisibleName *string `gorm:"index:,default:null"` + BinaryName *string `gorm:"index:,default:null"` + Version string + OriginalVersion string + Status string + Summary string + Active bool `gorm:"index:,default:false"` +} + +var RepologyProjectProviderColumns = struct { + ID string + ProjectName string + Repository string + SubRepository string + SourceName string + VisibleName string + BinaryName string + Version string + OriginalVersion string + Status string + Summary string + Active string +}{ + ID: "id", + ProjectName: "project_name", + Repository: "repository", + SubRepository: "sub_repository", + SourceName: "source_name", + VisibleName: "visible_name", + BinaryName: "binary_name", + Version: "version", + OriginalVersion: "original_version", + Status: "status", + Summary: "summary", + Active: "active", +} diff --git a/server/repology/dto.go b/server/repology/dto.go deleted file mode 100644 index 1933cadb..00000000 --- a/server/repology/dto.go +++ /dev/null @@ -1,28 +0,0 @@ -package repology - -const ( - repologProjectUrl = "https://repology.org/api/v1/project/%s" -) - -type repologyProject struct { - PrettyName string - Version string -} - -type repologyRawProject = map[string]interface{} - -type repologySemiRawProject struct { - Version string - VisibleName string - Name string - Status string - SrcName string - BinName string - Repo string - SubRepo string - Licenses []string - OrigVersion string - Summary string - Maintainers []string - Categories []string -} diff --git a/server/repology/exporter.go b/server/repology/exporter.go new file mode 100644 index 00000000..177add9f --- /dev/null +++ b/server/repology/exporter.go @@ -0,0 +1,5 @@ +package repology + +import "pacstall.dev/webserver/repology/internal" + +var ExportRepologyDatabase = internal.ExportRepologyDatabase diff --git a/server/repology/fetch.go b/server/repology/fetch.go index 32085f81..0bb996fe 100644 --- a/server/repology/fetch.go +++ b/server/repology/fetch.go @@ -1,158 +1,78 @@ package repology import ( - "encoding/json" + "errors" "fmt" - "io/ioutil" - "net/http" "strings" - "time" - "github.com/hashicorp/go-version" + "pacstall.dev/webserver/model" "pacstall.dev/webserver/types/list" ) -var repologyCache = make(map[string][]repologyRawProject) -var cachedUpdatedAt = time.Now() - -func fetchRaw(project string) ([]repologyRawProject, error) { - if cachedUpdatedAt.Add(3*time.Minute).After(time.Now()) && repologyCache[project] != nil { - return repologyCache[project], nil - } - - resp, err := http.Get(fmt.Sprintf(repologProjectUrl, project)) - if err != nil { - return nil, fmt.Errorf("(%v) Failed to fetch repology project: %v", project, err) - } - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("(%v) Failed with status %v to fetch repology project: %v", project, resp.StatusCode, err) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("Failed to read repology response: %v", err) - } - - result := make([]repologyRawProject, 0) - err = json.Unmarshal(body, &result) - if err != nil { - return nil, fmt.Errorf("Failed to unmarshal repology response: %v. \n\n%v\n\n", string(body), err) - } - - repologyCache[project] = result - cachedUpdatedAt = time.Now() - - if len(result) == 0 { - return nil, fmt.Errorf("No results for '%v'", project) - } - - return result, nil -} - -func getProperty(project repologyRawProject, property string) string { - if project[property] == nil { - return "" - } - - return project[property].(string) -} - -func getSliceProperty(project repologyRawProject, property string) []string { - if project[property] == nil { - return nil - } - - return list.Map(project[property].([]interface{}), func(i int, t interface{}) string { - return fmt.Sprintf("%v", t) - }) -} - func parseRepologyFilter(filter string) (string, string) { idx := strings.Index(filter, ":") return strings.TrimSpace(filter[:idx]), strings.TrimSpace(filter[idx+1:]) } -func fetchRepologyProject(search []string) (rpProj repologyProject, err error) { - _, project := parseRepologyFilter(search[0]) - result, err := fetchRaw(project) - if err != nil { - return - } +var repologyFilterToColumn = map[string]string{ + "repo": model.RepologyProjectProviderColumns.Repository, + "subrepo": model.RepologyProjectProviderColumns.SubRepository, + "status": model.RepologyProjectProviderColumns.Status, + "srcname": model.RepologyProjectProviderColumns.SourceName, + "binname": model.RepologyProjectProviderColumns.BinaryName, + "version": model.RepologyProjectProviderColumns.Version, + "origversion": model.RepologyProjectProviderColumns.OriginalVersion, + "visiblename": model.RepologyProjectProviderColumns.VisibleName, + "summary": model.RepologyProjectProviderColumns.Summary, +} - propertyPairs := list.Map(list.From(search[1:]), func(_ int, t string) []string { - filterName, filterValue := parseRepologyFilter(t) - return []string{filterName, filterValue} - }) +func findRepologyProject(search []string) (model.RepologyProjectProvider, error) { + var result model.RepologyProjectProvider - foundPackagesRaw := list.Map(list.From(result).Filter(func(pkg repologyRawProject) bool { - return list.From(propertyPairs).All(func(pair []string) bool { - return pkg[pair[0]] == pair[1] && strings.ContainsAny(pkg["version"].(string), "1234567890") - }) - }), func(i int, t repologyRawProject) repologySemiRawProject { - return repologySemiRawProject{ - Name: getProperty(t, "name"), - Version: getProperty(t, "version"), - VisibleName: getProperty(t, "visiblename"), - Summary: getProperty(t, "summary"), - Repo: getProperty(t, "repo"), - Status: getProperty(t, "status"), - SrcName: getProperty(t, "srcname"), - BinName: getProperty(t, "binname"), - SubRepo: getProperty(t, "subrepo"), - Licenses: getSliceProperty(t, "licenses"), - OrigVersion: getProperty(t, "origversion"), - Maintainers: getSliceProperty(t, "maintainers"), - Categories: getSliceProperty(t, "categories"), - } - }).Filter(func(pkg repologySemiRawProject) bool { - return pkg.Status != "incorrect" - }).SortBy(func(p1, p2 repologySemiRawProject) bool { - v1HasNumbers := strings.ContainsAny(p1.Version, "0123456789") - v2HasNumbers := strings.ContainsAny(p2.Version, "0123456789") - - if v1HasNumbers && !v2HasNumbers { - return true - } else if !v1HasNumbers && v2HasNumbers { - return false - } else if !v1HasNumbers && !v2HasNumbers { - return true - } + if len(search) == 0 { + return result, fmt.Errorf("no search terms provided") + } - v1, err := version.NewVersion(p1.Version) - if err != nil { - return false - } + db := model.Instance() + _, projectName := parseRepologyFilter(search[0]) - v2, err := version.NewVersion(p2.Version) - if err != nil { - return true + query := db.Where("project_name = ?", projectName).Where(fmt.Sprintf("%v = ?", model.RepologyProjectProviderColumns.Active), true) + for _, filter := range search[1:] { + filterName, filterValue := parseRepologyFilter(filter) + column, ok := repologyFilterToColumn[filterName] + if !ok { + return result, fmt.Errorf("invalid filter '%v'", filterName) } - return v1.GreaterThan(v2) - }).SortBy(func(rsrp1, rsrp2 repologySemiRawProject) bool { - return rsrp1.Status == "newest" && rsrp2.Status != "newest" - }) - - if foundPackagesRaw.Len() == 0 { - return rpProj, fmt.Errorf("no results for '%v' after applying search constraints", project) + query = query.Where(fmt.Sprintf("%v = ?", column), filterValue) } - rpProj.Version = foundPackagesRaw[0].Version - rpProj.PrettyName = foundPackagesRaw[0].VisibleName - - if strings.ToLower(rpProj.PrettyName) != rpProj.PrettyName { - return + var results []model.RepologyProjectProvider + if err := query.Order("version desc").Find(&results).Error; err != nil || len(results) == 0 { + return result, errors.Join(errors.New("failed to fetch repology project"), err) } - kindaPrettyList := foundPackagesRaw.Filter(func(p repologySemiRawProject) bool { - return strings.ToLower(p.VisibleName) != p.VisibleName - }) + results = sortByStatus(results) + result = results[0] - if kindaPrettyList.IsEmpty() { - return - } + return result, nil +} + +var repologyStatusPriority = map[string]int{ + "newest": 0, + "rolling": 1, + "devel": 3, + "legacy": 4, + "outdated": 5, + "unique": 6, + "noscheme": 7, + "incorrect": 7, + "untrusted": 7, + "ignored": 7, +} - rpProj.PrettyName = kindaPrettyList[0].VisibleName - return +func sortByStatus(projects []model.RepologyProjectProvider) []model.RepologyProjectProvider { + return list.From(projects).SortBy(func(p1, p2 model.RepologyProjectProvider) bool { + return repologyStatusPriority[p1.Status] < repologyStatusPriority[p2.Status] + }).ToSlice() } diff --git a/server/repology/internal/api.go b/server/repology/internal/api.go new file mode 100644 index 00000000..bbc3cee9 --- /dev/null +++ b/server/repology/internal/api.go @@ -0,0 +1,40 @@ +package internal + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/url" +) + +const _USER_AGENT = "Pacstall/WebServer/Exporter" + +func getProjectSearch(projectName string) (RepologyApiProjectSearchResponse, error) { + var response RepologyApiProjectSearchResponse + + request := http.Request{ + Method: "GET", + URL: &url.URL{Scheme: "https", Host: "repology.org", Path: "/api/v1/projects/" + projectName}, + Header: map[string][]string{ + "Accept": {"application/json"}, + "User-Agent": {_USER_AGENT}, + }, + } + + resp, err := http.DefaultClient.Do(&request) + if err != nil { + return response, err + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return response, err + } + + err = json.Unmarshal(body, &response) + if err != nil { + return response, err + } + + return response, err +} diff --git a/server/repology/internal/api_types.go b/server/repology/internal/api_types.go new file mode 100644 index 00000000..57d48214 --- /dev/null +++ b/server/repology/internal/api_types.go @@ -0,0 +1,18 @@ +package internal + +type RepologyApiProject struct { + Repository string `json:"repo"` + SubRepository *string `json:"subrepo"` + SourceName *string `json:"srcname"` + VisibleName *string `json:"visiblename"` + BinaryName *string `json:"binname"` + Version string `json:"version"` + OriginalVersion string `json:"origversion"` + Status string `json:"status"` + Summary string `json:"summary"` + Licenses []string `json:"licenses"` + Maintainers []string `json:"maintainers"` + Categories []string `json:"categories"` +} + +type RepologyApiProjectSearchResponse = map[string][]RepologyApiProject diff --git a/server/repology/internal/exporter.go b/server/repology/internal/exporter.go new file mode 100644 index 00000000..4e986631 --- /dev/null +++ b/server/repology/internal/exporter.go @@ -0,0 +1,128 @@ +package internal + +import ( + "errors" + "fmt" + "sync" + "time" + + "gorm.io/gorm" + "pacstall.dev/webserver/log" + "pacstall.dev/webserver/model" +) + +func ExportRepologyDatabase(db *gorm.DB) error { + err := migrateTables(db) + if err != nil { + return errors.Join(errors.New("failed to reset repology tables"), err) + } + + it := 1 + lastProjectName := "" + for { + delay := makeSecondDelay() + defer delay.Wait() + + log.Debug("Page %v | Cursor at: %v", it, lastProjectName) + projectPage, err := getProjectSearch(lastProjectName) + if err != nil { + return errors.Join(errors.New("failed to fetch repology project page"), err) + } + + var projectProviders []model.RepologyProjectProvider + var projects []model.RepologyProject + for projectName, apiProjectProvider := range projectPage { + lastProjectName = projectName + for _, apiProjectProvider := range apiProjectProvider { + // Save project provider as inactive + projectProvider := mapRepologyApiProjectProviderToModel(projectName, apiProjectProvider) + projectProviders = append(projectProviders, projectProvider) + + project := model.RepologyProject{ + Name: projectName, + } + + projects = append(projects, project) + } + } + + if len(projects) <= 1 { + break + } + + err = db.Save(&projects).Error + if err != nil { + return errors.Join(errors.New("failed to create repology projects"), err) + } + + err = db.CreateInBatches(&projectProviders, 90).Error + if err != nil { + return errors.Join(errors.New("failed to create repology project providers"), err) + } + + it++ + } + + // Delete active (old) repology project providers + if err := db.Where(fmt.Sprintf("%v = ?", model.RepologyProjectProviderColumns.Active), true).Delete(&model.RepologyProjectProvider{}).Error; err != nil { + return errors.Join(errors.New("failed to delete old repology project providers"), err) + } + + // Mark new repology project providers as active + if err := db.Exec( + fmt.Sprintf( + "UPDATE %s SET %s = 1", + model.RepologyProjectProviderTableName, + model.RepologyProjectProviderColumns.Active, + ), + ).Error; err != nil { + return errors.Join(errors.New("failed to update new repology project providers"), err) + } + + return nil +} + +func migrateTables(db *gorm.DB) error { + err := db.AutoMigrate(&model.RepologyProject{}) + if err != nil { + return err + } + + err = db.AutoMigrate(&model.RepologyProjectProvider{}) + if err != nil { + return err + } + + return nil +} + +func makeSecondDelay() *sync.WaitGroup { + var delay sync.WaitGroup + delay.Add(1) + + go func() { + defer delay.Done() + // Wait 750ms before making another request + // Repology API has a rate limit of 1 request per second but some requests take longer than 1 second so it averages out + time.Sleep(750 * time.Millisecond) + }() + + return &delay +} + +func mapRepologyApiProjectProviderToModel(projectName string, apiProjectProvider RepologyApiProject) model.RepologyProjectProvider { + projectProvider := model.RepologyProjectProvider{ + ProjectName: projectName, + Repository: apiProjectProvider.Repository, + SubRepository: apiProjectProvider.SubRepository, + SourceName: apiProjectProvider.SourceName, + VisibleName: apiProjectProvider.VisibleName, + BinaryName: apiProjectProvider.BinaryName, + Version: apiProjectProvider.Version, + OriginalVersion: apiProjectProvider.OriginalVersion, + Status: apiProjectProvider.Status, + Summary: apiProjectProvider.Summary, + Active: false, + } + return projectProvider +} diff --git a/server/repology/lib.go b/server/repology/lib.go index bef55b05..9153cdc1 100644 --- a/server/repology/lib.go +++ b/server/repology/lib.go @@ -9,10 +9,10 @@ func Sync(script *pac.Script) error { return nil } - project, err := fetchRepologyProject(script.Repology) + project, err := findRepologyProject(script.Repology) if err != nil { return err } - return syncToPacscript(project, script) + return updateScriptVersion(project, script) } diff --git a/server/repology/retry.go b/server/repology/retry.go deleted file mode 100644 index 82d54571..00000000 --- a/server/repology/retry.go +++ /dev/null @@ -1,48 +0,0 @@ -package repology - -import ( - "fmt" - "time" - - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/types/pac" -) - -func NewSyncer(maxRetries int) func(*pac.Script) error { - baseTime := time.Second * 5 - multiplier := 0.2 - retries := maxRetries - - return func(script *pac.Script) error { - defer func() { - retries = maxRetries - }() - - for retries > 0 { - - if multiplier <= 0 { - multiplier = 1 - } - - computedDelay := baseTime * time.Duration(multiplier) - - retries -= 1 - time.Sleep(computedDelay) - - if retries < maxRetries-1 { - log.Debug("Trying to sync with repology. Delay %v, Delay Multiplier %.2f", computedDelay, multiplier) - } - - if err := Sync(script); err != nil { - log.Debug("Failed to fetch repology information. Increasing delay. %v", err) - multiplier *= 1.5 - continue - } - - multiplier *= 0.9 - return nil - } - - return fmt.Errorf("failed to fetch repology information after %v retries", maxRetries) - } -} diff --git a/server/repology/scheduler.go b/server/repology/scheduler.go new file mode 100644 index 00000000..a4423c6d --- /dev/null +++ b/server/repology/scheduler.go @@ -0,0 +1,25 @@ +package repology + +import ( + "time" + + "pacstall.dev/webserver/log" + "pacstall.dev/webserver/model" +) + +func ScheduleRefresh(every time.Duration) { + go func() { + for { + db := model.Instance() + log.Info("Refreshing Repology database...") + err := ExportRepologyDatabase(db) + if err != nil { + log.Error("Failed to export Repology projects: %v", err) + } else { + log.Info("Repology database refreshed successfully") + } + + time.Sleep(every) + } + }() +} diff --git a/server/repology/sync.go b/server/repology/sync.go index 29fc0a74..c1bd83c8 100644 --- a/server/repology/sync.go +++ b/server/repology/sync.go @@ -4,11 +4,24 @@ import ( "strings" "github.com/hashicorp/go-version" + "pacstall.dev/webserver/model" "pacstall.dev/webserver/types/pac" ) -func syncToPacscript(project repologyProject, script *pac.Script) (err error) { - script.PrettyName = project.PrettyName +func compareNonStandardVersion(current, latest string) pac.UpdateStatusValue { + result := strings.Compare(current, latest) + + const CMP_EQUAL = 0 + const CMP_GREATER = 1 + + if result == CMP_EQUAL || result == CMP_GREATER { + return pac.UpdateStatus.Latest + } + + return pac.UpdateStatus.Major +} + +func updateScriptVersion(project model.RepologyProjectProvider, script *pac.Script) (err error) { script.LatestVersion = &project.Version if *script.LatestVersion == script.Version { @@ -19,14 +32,14 @@ func syncToPacscript(project repologyProject, script *pac.Script) (err error) { current, err := version.NewVersion(script.Version) if err != nil { err = nil - script.UpdateStatus = versionCompare(script.Version, *script.LatestVersion) + script.UpdateStatus = compareNonStandardVersion(script.Version, *script.LatestVersion) return } latest, err := version.NewVersion(*script.LatestVersion) if err != nil { err = nil - script.UpdateStatus = versionCompare(script.Version, *script.LatestVersion) + script.UpdateStatus = compareNonStandardVersion(script.Version, *script.LatestVersion) return } diff --git a/server/types/pac/parser/parse.go b/server/types/pac/parser/parse.go index 2ca3fe1f..94290898 100644 --- a/server/types/pac/parser/parse.go +++ b/server/types/pac/parser/parse.go @@ -71,26 +71,17 @@ func parsePacscriptFiles(names []string) []*pac.Script { if err != nil { log.Warn("Failed to parse %v. err: %v", pacName, err) } - return &out, err - }) - - results := channels.ToSlice(outChan) - - if !config.Repology.Enabled { - return results - } - repologySync := repology.NewSyncer(15) - log.Info("Syncing pacscripts with repology...") - - for _, result := range results { - log.Info("Checking %v", result.Name) - if err := repologySync(result); err != nil { - log.Warn("Failed to sync %v. err: %v", result.Name, err) + if config.Repology.Enabled { + if err := repology.Sync(&out); err != nil { + log.Debug("Failed to sync %v with repology. Error: %v", pacName, err) + } } - } - return results + return &out, err + }) + + return channels.ToSlice(outChan) } func readPacscriptFile(rootDir, name string) (scriptBytes []byte, fileName string, err error) { diff --git a/server/types/pac/parser/scheduler.go b/server/types/pac/parser/scheduler.go index bcabb0fc..9700692d 100644 --- a/server/types/pac/parser/scheduler.go +++ b/server/types/pac/parser/scheduler.go @@ -9,12 +9,12 @@ import ( func ScheduleRefresh(every time.Duration) { go func() { for { - time.Sleep(every) err := ParseAll() if err != nil { log.Error("Failed to parse pacscripts: %v", err) } + time.Sleep(every) } }() }