feat: add repology exporter (#498)

Co-authored-by: Elsie <[email protected]>

server/Makefile

@@ -28,6 +28,7 @@ run:
  PACSTALL_MATOMO_ENABLED="false" \
  PACSTALL_REPOLOGY_ENABLED="false" \
  go run bin/webserver/main.go
+
 dist/webserver: $(shell find . -not \( -path ./tmp -prune \) -not \( -path ./dist -prune \) -type f)
  CGO_ENABLED=0 go build -o dist/webserver -ldflags "${LDFLAGS}" bin/webserver/main.go
 clean:

server/bin/webserver/main.go

@@ -7,6 +7,7 @@ import (
  "github.com/fatih/color"
  "pacstall.dev/webserver/config"
  "pacstall.dev/webserver/log"
+ "pacstall.dev/webserver/repology"
  "pacstall.dev/webserver/server"
  ps_api "pacstall.dev/webserver/server/api/pacscripts"
  repology_api "pacstall.dev/webserver/server/api/repology"
@@ -54,8 +55,6 @@ func main() {
  }
 
  startedAt := time.Now()
- port := config.Port
- refreshTimer := config.UpdateInterval
 
  printLogo()
 
@@ -66,19 +65,20 @@ func main() {
 
  server.OnServerOnline(func() {
  log.NotifyCustom("🚀 Startup 🧑‍🚀", "Successfully started up.")
- log.Info("Server is now online on port %v.\n", port)
+ log.Info("Server is now online on port %v.\n", config.Port)
 
  log.Info("Booted in %v\n", color.GreenString("%v", time.Since(startedAt)))
 
- log.Info("Attempting to parse existing pacscripts")
- err := parser.ParseAll()
- if err != nil {
- log.Error("Failed to parse pacscripts: %v", err)
- }
+ parser.ScheduleRefresh(config.UpdateInterval)
+ log.Info("Scheduled pacscripts to auto-refresh every %v", config.UpdateInterval)
 
- parser.ScheduleRefresh(refreshTimer)
- log.Info("Scheduled pacscripts to auto-refresh every %v", refreshTimer)
+ if config.Repology.Enabled {
+ repology.ScheduleRefresh(config.RepologyUpdateInterval)
+ log.Info("Scheduled repology to auto-refresh every %v", config.RepologyUpdateInterval)
+ } else {
+ log.Warn("Repology is disabled. Pacstall will not be able to fetch package data from Repology.")
+ }
  })
 
- server.Listen(port)
+ server.Listen(config.Port)
 }

server/config/build/vars.go

@@ -3,6 +3,7 @@ package build
 var Production = "false"
 
 var UpdateInterval = "900"
+var RepologyUpdateInterval = "43200"
 var TempDir = "./tmp"
 var MaxOpenFiles = "100"
 var GitURL = "https://github.com/pacstall/pacstall-programs.git"

server/config/vars.go

@@ -9,6 +9,7 @@ import (
 var Production = toBool(build.Production)
 
 var UpdateInterval = time.Duration(toInt(build.UpdateInterval)) * time.Second
+var RepologyUpdateInterval = time.Duration(toInt(build.RepologyUpdateInterval)) * time.Second
 var TempDir = build.TempDir
 var MaxOpenFiles = toInt(build.MaxOpenFiles)
 var GitURL = build.GitURL

server/go.sum

@@ -48,14 +48,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/mysql v1.5.0 h1:6hSAT5QcyIaty0jfnff0z0CLDjyRgZ8mlMHLqSt7uXM=
-gorm.io/driver/mysql v1.5.0/go.mod h1:FFla/fJuCvyTi7rJQd27qlNX2v3L6deTR1GgTjSOLPo=
 gorm.io/driver/mysql v1.5.1 h1:WUEH5VF9obL/lTtzjmML/5e6VfFR/788coz2uaVCAZw=
 gorm.io/driver/mysql v1.5.1/go.mod h1:Jo3Xu7mMhCyj8dlrb3WoCaRd1FhsVh+yMXb1jUInf5o=
-gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
-gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU=
-gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
-gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64=
 gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
 gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho=
 gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=

server/model/connection.go

@@ -5,6 +5,7 @@ import (
 
  "gorm.io/driver/mysql"
  "gorm.io/gorm"
+ "gorm.io/gorm/logger"
  "pacstall.dev/webserver/config"
 )
 
@@ -22,7 +23,10 @@ func Instance() *gorm.DB {
  return database
  }
 
- db, err := gorm.Open(mysql.Open(connectionString), &gorm.Config{})
+ db, err := gorm.Open(mysql.Open(connectionString), &gorm.Config{
+ Logger: logger.Default.LogMode(logger.Silent),
+ })
+
  if err != nil {
  panic(fmt.Sprintf("failed to connect database: %v", err))
  }

server/model/repology_project.go

@@ -0,0 +1,11 @@
+package model
+
+type RepologyProject struct {
+ Name string `gorm:"primaryKey"`
+}
+
+var RepologyProjectColumns = struct {
+ Name string
+}{
+ Name: "name",
+}

server/model/repology_project_provider.go

@@ -0,0 +1,47 @@
+package model
+
+const RepologyProjectProviderTableName = "repology_project_providers"
+
+type RepologyProjectProvider struct {
+ ID uint `gorm:"primarykey"`
+ ProjectName string `gorm:"index:"`
+ Project RepologyProject `gorm:"foreignKey:Name"`
+ Repository string
+ SubRepository *string `gorm:"default:null"`
+ SourceName *string `gorm:"index:,default:null"`
+ VisibleName *string `gorm:"index:,default:null"`
+ BinaryName *string `gorm:"index:,default:null"`
+ Version string
+ OriginalVersion string
+ Status string
+ Summary string
+ Active bool `gorm:"index:,default:false"`
+}
+
+var RepologyProjectProviderColumns = struct {
+ ID string
+ ProjectName string
+ Repository string
+ SubRepository string
+ SourceName string
+ VisibleName string
+ BinaryName string
+ Version string
+ OriginalVersion string
+ Status string
+ Summary string
+ Active string
+}{
+ ID: "id",
+ ProjectName: "project_name",
+ Repository: "repository",
+ SubRepository: "sub_repository",
+ SourceName: "source_name",
+ VisibleName: "visible_name",
+ BinaryName: "binary_name",
+ Version: "version",
+ OriginalVersion: "original_version",
+ Status: "status",
+ Summary: "summary",
+ Active: "active",
+}

server/repology/exporter.go

@@ -0,0 +1,5 @@
+package repology
+
+import "pacstall.dev/webserver/repology/internal"
+
+var ExportRepologyDatabase = internal.ExportRepologyDatabase