From a53c0293274a609c2f73274bab73469e345f5c06 Mon Sep 17 00:00:00 2001 From: moqsien Date: Tue, 5 Mar 2024 10:11:37 +0800 Subject: [PATCH] update filter --- pkgs/versions/github.go | 30 +++++++++--------- pkgs/versions/installers.go | 20 +++++++++++- pkgs/versions/julia.go | 62 ++++++++++++++++++++++++------------- pkgs/versions/nodejs.go | 8 +++-- pkgs/versions/php.go | 4 +++ 5 files changed, 86 insertions(+), 38 deletions(-) diff --git a/pkgs/versions/github.go b/pkgs/versions/github.go index 7cdcbc3..9642fe3 100644 --- a/pkgs/versions/github.go +++ b/pkgs/versions/github.go @@ -76,20 +76,22 @@ func filterByUrl(dUrl string) bool { ".txt", ".d.ts", "src.tar.gz", - "-baseline.zip", // for bun - "-profile.zip", // for bun - "denort-", // for deno - "-unknown-linux-musl.tar.gz", // for fd. - "-pc-windows-gnu.zip", // for fd. - "linux-gnueabihf", // for fd - "linux-musleabihf", // for fd - "-musl-x86_64", // for julia -- untested - "-win64.tar.gz", // for julia - "-win32.tar.gz", // for julia - "kotlin-compiler-", // for kotlin - "Miniconda2-latest-", // for miniconda - "Miniconda-latest-", // for miniconda - // nodejs + "-baseline.zip", // for bun + "-profile.zip", // for bun + "denort-", // for deno + "-unknown-linux-musl.tar.gz", // for fd. + "-pc-windows-gnu.zip", // for fd. + "linux-gnueabihf", // for fd + "linux-musleabihf", // for fd + "kotlin-compiler-", // for kotlin + "unknown-linux-gnueabihf.", // for ripgrep + "unknown-linux-musleabi.", // for ripgrep + "unknown-linux-musleabihf.", // for ripgrep + "pc-windows-gnu.zip", // for ripgrep + "arm-unknown-linux-gnueabihf", // for typst-lsp + "typst-lsp-x86_64-unknown-linux-musl", // for typst-lsp + "-unknown-linux-musleabi.", // for typst + "wasm32-wasi.", // for zls } for _, s := range excludeList { if strings.Contains(dUrl, s) { diff --git a/pkgs/versions/installers.go b/pkgs/versions/installers.go index 619f299..2117fb0 100644 --- a/pkgs/versions/installers.go +++ b/pkgs/versions/installers.go @@ -252,6 +252,21 @@ func (i *Installer) GetVSCode() { } } +func filterMinicondaByFName(fname string) bool { + r := false + toBunList := []string{ + ".pkg", + "Miniconda2-latest-", // for miniconda + "Miniconda-latest-", // for miniconda + } + for _, b := range toBunList { + if strings.Contains(fname, b) { + return true + } + } + return r +} + func (i *Installer) GetMiniconda() { // https://repo.anaconda.com/miniconda/ i.homepage = "https://repo.anaconda.com/miniconda/" @@ -271,7 +286,10 @@ func (i *Installer) GetMiniconda() { return } fName := s.Find("td").Eq(0).Find("a").Text() - if strings.HasSuffix(fName, ".pkg") { + // if strings.HasSuffix(fName, ".pkg") { + // return + // } + if filterMinicondaByFName(fName) { return } sha256Str := s.Find("td").Eq(3).Text() diff --git a/pkgs/versions/julia.go b/pkgs/versions/julia.go index fb8a9ab..1f3eb12 100644 --- a/pkgs/versions/julia.go +++ b/pkgs/versions/julia.go @@ -10,11 +10,11 @@ import ( "time" "github.com/gogf/gf/v2/util/gconv" - "github.com/gvcgo/goutils/pkgs/gtea/gprint" - "github.com/gvcgo/goutils/pkgs/request" "github.com/gvcgo/collector/pkgs/confs" "github.com/gvcgo/collector/pkgs/upload" "github.com/gvcgo/collector/pkgs/utils" + "github.com/gvcgo/goutils/pkgs/gtea/gprint" + "github.com/gvcgo/goutils/pkgs/request" ) const ( @@ -69,6 +69,22 @@ func NewJulia(cnf *confs.CollectorConf) (j *Julia) { return } +func filterJuliaByUrl(jUrl string) bool { + r := false + toBunList := []string{ + "-musl-x86_64", // for julia -- untested + "-win64.tar.gz", // for julia + "-win32.tar.gz", // for julia + ".dmg", + } + for _, b := range toBunList { + if strings.Contains(jUrl, b) { + return true + } + } + return r +} + func (j *Julia) GetVersions() { j.fetcher.SetUrl(j.homepage) j.fetcher.Timeout = 180 * time.Second @@ -87,26 +103,30 @@ func (j *Julia) GetVersions() { extraStr = "stable" } for _, jFile := range fList.Files { - if jFile.Kind == "archive" && !strings.HasSuffix(jFile.Url, ".dmg") { - archStr := utils.ParseArch(jFile.Arch) - platform := utils.ParsePlatform(jFile.Os) - if archStr == "" || platform == "" { - continue - } - ver := &VFile{} - ver.Url = jFile.Url - ver.Arch = archStr - ver.Os = platform - ver.Sum = jFile.Sum - if ver.Sum != "" { - ver.SumType = "sha256" - } - ver.Extra = extraStr - if vlist, ok := j.versions[vName]; !ok || vlist == nil { - j.versions[vName] = []*VFile{} - } - j.versions[vName] = append(j.versions[vName], ver) + if jFile.Kind != "archive" { + continue + } + if filterJuliaByUrl(jFile.Url) { + continue + } + archStr := utils.ParseArch(jFile.Arch) + platform := utils.ParsePlatform(jFile.Os) + if archStr == "" || platform == "" { + continue + } + ver := &VFile{} + ver.Url = jFile.Url + ver.Arch = archStr + ver.Os = platform + ver.Sum = jFile.Sum + if ver.Sum != "" { + ver.SumType = "sha256" + } + ver.Extra = extraStr + if vlist, ok := j.versions[vName]; !ok || vlist == nil { + j.versions[vName] = []*VFile{} } + j.versions[vName] = append(j.versions[vName], ver) } } } diff --git a/pkgs/versions/nodejs.go b/pkgs/versions/nodejs.go index 8390d9a..eaeab30 100644 --- a/pkgs/versions/nodejs.go +++ b/pkgs/versions/nodejs.go @@ -11,11 +11,11 @@ import ( "time" "github.com/gogf/gf/v2/util/gconv" - "github.com/gvcgo/goutils/pkgs/gtea/gprint" - "github.com/gvcgo/goutils/pkgs/request" "github.com/gvcgo/collector/pkgs/confs" "github.com/gvcgo/collector/pkgs/upload" "github.com/gvcgo/collector/pkgs/utils" + "github.com/gvcgo/goutils/pkgs/gtea/gprint" + "github.com/gvcgo/goutils/pkgs/request" ) const ( @@ -83,6 +83,10 @@ func (n *Nodejs) getVersion(vItem *Item) { uList := strings.Split(strings.TrimSpace(line), " ") if len(uList) >= 2 { fName := strings.TrimSpace(uList[len(uList)-1]) + // filter + if strings.Contains(fName, "node_pdb.zip") { + continue + } archStr := utils.ParseArch(fName) osStr := utils.ParsePlatform(fName) if archStr != "" && osStr != "" { diff --git a/pkgs/versions/php.go b/pkgs/versions/php.go index 9419652..b7b8db0 100644 --- a/pkgs/versions/php.go +++ b/pkgs/versions/php.go @@ -113,6 +113,10 @@ func (p *PhP) GetWindowsVersions() { } ver := &VFile{} u := s.AttrOr("href", "") + // filter: discard Non-safe-thread version for Windows Server. + if strings.Contains(u, "nts-Win32") { + return + } if !strings.HasPrefix(u, "https://") { u, _ = url.JoinPath(baseUrl, u) }