update filter

gvcgo · Mar 5, 2024 · a53c029 · a53c029
1 parent ab59d7f
commit a53c029
Show file tree

Hide file tree

Showing 5 changed files with 86 additions and 38 deletions.
diff --git a/pkgs/versions/github.go b/pkgs/versions/github.go
@@ -76,20 +76,22 @@ func filterByUrl(dUrl string) bool {
 		".txt",
 		".d.ts",
 		"src.tar.gz",
-		"-baseline.zip",              // for bun
-		"-profile.zip",               // for bun
-		"denort-",                    // for deno
-		"-unknown-linux-musl.tar.gz", // for fd.
-		"-pc-windows-gnu.zip",        // for fd.
-		"linux-gnueabihf",            // for fd
-		"linux-musleabihf",           // for fd
-		"-musl-x86_64",               // for julia -- untested
-		"-win64.tar.gz",              // for julia
-		"-win32.tar.gz",              // for julia
-		"kotlin-compiler-",           // for kotlin
-		"Miniconda2-latest-",         // for miniconda
-		"Miniconda-latest-",          // for miniconda
-		// nodejs
+		"-baseline.zip",                       // for bun
+		"-profile.zip",                        // for bun
+		"denort-",                             // for deno
+		"-unknown-linux-musl.tar.gz",          // for fd.
+		"-pc-windows-gnu.zip",                 // for fd.
+		"linux-gnueabihf",                     // for fd
+		"linux-musleabihf",                    // for fd
+		"kotlin-compiler-",                    // for kotlin
+		"unknown-linux-gnueabihf.",            // for ripgrep
+		"unknown-linux-musleabi.",             // for ripgrep
+		"unknown-linux-musleabihf.",           // for ripgrep
+		"pc-windows-gnu.zip",                  // for ripgrep
+		"arm-unknown-linux-gnueabihf",         // for typst-lsp
+		"typst-lsp-x86_64-unknown-linux-musl", // for typst-lsp
+		"-unknown-linux-musleabi.",            // for typst
+		"wasm32-wasi.",                        // for zls
 	}
 	for _, s := range excludeList {
 		if strings.Contains(dUrl, s) {

diff --git a/pkgs/versions/installers.go b/pkgs/versions/installers.go
@@ -252,6 +252,21 @@ func (i *Installer) GetVSCode() {
 	}
 }
 
+func filterMinicondaByFName(fname string) bool {
+	r := false
+	toBunList := []string{
+		".pkg",
+		"Miniconda2-latest-", // for miniconda
+		"Miniconda-latest-",  // for miniconda
+	}
+	for _, b := range toBunList {
+		if strings.Contains(fname, b) {
+			return true
+		}
+	}
+	return r
+}
+
 func (i *Installer) GetMiniconda() {
 	// https://repo.anaconda.com/miniconda/
 	i.homepage = "https://repo.anaconda.com/miniconda/"
@@ -271,7 +286,10 @@ func (i *Installer) GetMiniconda() {
 				return
 			}
 			fName := s.Find("td").Eq(0).Find("a").Text()
-			if strings.HasSuffix(fName, ".pkg") {
+			// if strings.HasSuffix(fName, ".pkg") {
+			// 	return
+			// }
+			if filterMinicondaByFName(fName) {
 				return
 			}
 			sha256Str := s.Find("td").Eq(3).Text()

diff --git a/pkgs/versions/julia.go b/pkgs/versions/julia.go
@@ -10,11 +10,11 @@ import (
 	"time"
 
 	"github.com/gogf/gf/v2/util/gconv"
-	"github.com/gvcgo/goutils/pkgs/gtea/gprint"
-	"github.com/gvcgo/goutils/pkgs/request"
 	"github.com/gvcgo/collector/pkgs/confs"
 	"github.com/gvcgo/collector/pkgs/upload"
 	"github.com/gvcgo/collector/pkgs/utils"
+	"github.com/gvcgo/goutils/pkgs/gtea/gprint"
+	"github.com/gvcgo/goutils/pkgs/request"
 )
 
 const (
@@ -69,6 +69,22 @@ func NewJulia(cnf *confs.CollectorConf) (j *Julia) {
 	return
 }
 
+func filterJuliaByUrl(jUrl string) bool {
+	r := false
+	toBunList := []string{
+		"-musl-x86_64",  // for julia -- untested
+		"-win64.tar.gz", // for julia
+		"-win32.tar.gz", // for julia
+		".dmg",
+	}
+	for _, b := range toBunList {
+		if strings.Contains(jUrl, b) {
+			return true
+		}
+	}
+	return r
+}
+
 func (j *Julia) GetVersions() {
 	j.fetcher.SetUrl(j.homepage)
 	j.fetcher.Timeout = 180 * time.Second
@@ -87,26 +103,30 @@ func (j *Julia) GetVersions() {
 				extraStr = "stable"
 			}
 			for _, jFile := range fList.Files {
-				if jFile.Kind == "archive" && !strings.HasSuffix(jFile.Url, ".dmg") {
-					archStr := utils.ParseArch(jFile.Arch)
-					platform := utils.ParsePlatform(jFile.Os)
-					if archStr == "" || platform == "" {
-						continue
-					}
-					ver := &VFile{}
-					ver.Url = jFile.Url
-					ver.Arch = archStr
-					ver.Os = platform
-					ver.Sum = jFile.Sum
-					if ver.Sum != "" {
-						ver.SumType = "sha256"
-					}
-					ver.Extra = extraStr
-					if vlist, ok := j.versions[vName]; !ok || vlist == nil {
-						j.versions[vName] = []*VFile{}
-					}
-					j.versions[vName] = append(j.versions[vName], ver)
+				if jFile.Kind != "archive" {
+					continue
+				}
+				if filterJuliaByUrl(jFile.Url) {
+					continue
+				}
+				archStr := utils.ParseArch(jFile.Arch)
+				platform := utils.ParsePlatform(jFile.Os)
+				if archStr == "" || platform == "" {
+					continue
+				}
+				ver := &VFile{}
+				ver.Url = jFile.Url
+				ver.Arch = archStr
+				ver.Os = platform
+				ver.Sum = jFile.Sum
+				if ver.Sum != "" {
+					ver.SumType = "sha256"
+				}
+				ver.Extra = extraStr
+				if vlist, ok := j.versions[vName]; !ok || vlist == nil {
+					j.versions[vName] = []*VFile{}
 				}
+				j.versions[vName] = append(j.versions[vName], ver)
 			}
 		}
 	}

diff --git a/pkgs/versions/nodejs.go b/pkgs/versions/nodejs.go
@@ -11,11 +11,11 @@ import (
 	"time"
 
 	"github.com/gogf/gf/v2/util/gconv"
-	"github.com/gvcgo/goutils/pkgs/gtea/gprint"
-	"github.com/gvcgo/goutils/pkgs/request"
 	"github.com/gvcgo/collector/pkgs/confs"
 	"github.com/gvcgo/collector/pkgs/upload"
 	"github.com/gvcgo/collector/pkgs/utils"
+	"github.com/gvcgo/goutils/pkgs/gtea/gprint"
+	"github.com/gvcgo/goutils/pkgs/request"
 )
 
 const (
@@ -83,6 +83,10 @@ func (n *Nodejs) getVersion(vItem *Item) {
 			uList := strings.Split(strings.TrimSpace(line), " ")
 			if len(uList) >= 2 {
 				fName := strings.TrimSpace(uList[len(uList)-1])
+				// filter
+				if strings.Contains(fName, "node_pdb.zip") {
+					continue
+				}
 				archStr := utils.ParseArch(fName)
 				osStr := utils.ParsePlatform(fName)
 				if archStr != "" && osStr != "" {

diff --git a/pkgs/versions/php.go b/pkgs/versions/php.go
@@ -113,6 +113,10 @@ func (p *PhP) GetWindowsVersions() {
 			}
 			ver := &VFile{}
 			u := s.AttrOr("href", "")
+			// filter: discard Non-safe-thread version for Windows Server.
+			if strings.Contains(u, "nts-Win32") {
+				return
+			}
 			if !strings.HasPrefix(u, "https://") {
 				u, _ = url.JoinPath(baseUrl, u)
 			}