From 165b315b3b4ac5544554f066b2ec61af88391612 Mon Sep 17 00:00:00 2001 From: Nitishkumar Singh Date: Sat, 22 Jun 2024 21:19:13 +0530 Subject: [PATCH] include system url as template Signed-off-by: Nitishkumar Singh move powershell and go to tools Signed-off-by: Nitishkumar Singh empty commit Signed-off-by: Nitishkumar Singh --- cmd/get.go | 2 +- cmd/system/go.go | 84 +++--------------- cmd/system/powershell.go | 53 +++--------- pkg/get/download.go | 54 ++++++++++++ pkg/get/get.go | 53 +++++++++++- pkg/get/get_test.go | 166 ++++++++++++++++++++++++++++++++++++ pkg/get/tools.go | 73 ++++++++++++++++ pkg/get/url_checker_test.go | 2 + 8 files changed, 372 insertions(+), 115 deletions(-) diff --git a/cmd/get.go b/cmd/get.go index 7d4b23b0a..f61d08e7a 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -24,7 +24,7 @@ import ( // MakeGet creates the Get command to download software func MakeGet() *cobra.Command { - tools := get.MakeTools() + tools := get.MakeToolsWithoutSystemApp() sort.Sort(tools) var validToolOptions []string = make([]string, len(tools)) for _, t := range tools { diff --git a/cmd/system/go.go b/cmd/system/go.go index deed1f799..d0332d9ce 100644 --- a/cmd/system/go.go +++ b/cmd/system/go.go @@ -5,14 +5,10 @@ package system import ( "fmt" - "io" - "net/http" "os" "path" "strings" - "github.com/alexellis/arkade/pkg" - "github.com/alexellis/arkade/pkg/archive" "github.com/alexellis/arkade/pkg/env" "github.com/alexellis/arkade/pkg/get" "github.com/spf13/cobra" @@ -53,51 +49,26 @@ func MakeInstallGo() *cobra.Command { return fmt.Errorf("this app only supports Linux") } - dlArch := arch - if arch == "x86_64" { - dlArch = "amd64" - } else if arch == "aarch64" { - dlArch = "arm64" - } else if arch == "armv7" || arch == "armv7l" { - dlArch = "armv6l" - } - - if len(version) == 0 { - v, err := getGoVersion() - if err != nil { - return err + tools := get.MakeTools() + var tool *get.Tool + for _, t := range tools { + if t.Name == "go" { + tool = &t + break } - - version = v - } else if !strings.HasPrefix(version, "go") { - version = "go" + version } - fmt.Printf("Installing version: %s for: %s\n", version, dlArch) - - dlURL := fmt.Sprintf("https://go.dev/dl/%s.%s-%s.tar.gz", version, strings.ToLower(osVer), dlArch) - fmt.Printf("Downloading from: %s\n", dlURL) - - progress, _ := cmd.Flags().GetBool("progress") - outPath, err := get.DownloadFileP(dlURL, progress) - if err != nil { - return err + if tool == nil { + return fmt.Errorf("unable to find go definition") } - defer os.Remove(outPath) - - fmt.Printf("Downloaded to: %s\n", outPath) - f, err := os.OpenFile(outPath, os.O_RDONLY, 0644) + progress, _ := cmd.Flags().GetBool("progress") + err := get.DownloadNested(tool, arch, osVer, version, installPath, progress, !progress) if err != nil { return err } - defer f.Close() - fmt.Printf("Unpacking Go to: %s\n", path.Join(installPath, "go")) - - if err := archive.UntarNested(f, installPath, true, false); err != nil { - return err - } + fmt.Printf("Downloaded to: %sgo\n", installPath) fmt.Printf("\nexport PATH=$PATH:%s:$HOME/go/bin\n"+ "export GOPATH=$HOME/go/\n", path.Join(installPath, "go", "bin")) @@ -107,36 +78,3 @@ func MakeInstallGo() *cobra.Command { return command } - -func getGoVersion() (string, error) { - req, err := http.NewRequest(http.MethodGet, "https://go.dev/VERSION?m=text", nil) - if err != nil { - return "", err - } - - req.Header.Set("User-Agent", pkg.UserAgent()) - - res, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - - if res.Body == nil { - return "", fmt.Errorf("unexpected empty body") - } - - defer res.Body.Close() - body, _ := io.ReadAll(res.Body) - - if res.StatusCode != http.StatusOK { - return "", fmt.Errorf("unexpected status code: %d", res.StatusCode) - } - - content := strings.TrimSpace(string(body)) - version, _, ok := strings.Cut(content, "\n") - if !ok { - return "", fmt.Errorf("format unexpected: %q", content) - } - - return version, nil -} diff --git a/cmd/system/powershell.go b/cmd/system/powershell.go index c6b136014..bc854cd97 100644 --- a/cmd/system/powershell.go +++ b/cmd/system/powershell.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/Masterminds/semver" - "github.com/alexellis/arkade/pkg/archive" "github.com/alexellis/arkade/pkg/env" "github.com/alexellis/arkade/pkg/get" "github.com/spf13/cobra" @@ -48,17 +47,21 @@ func MakeInstallPowershell() *cobra.Command { arch, _ = cmd.Flags().GetString("arch") } - dlArch := arch - if arch == "x86_64" { - dlArch = "x64" - } else if arch == "aarch64" { - dlArch = "arm64" - } else if arch == "armv7" || arch == "armv7l" { - dlArch = "arm32" + tools := get.MakeTools() + var tool *get.Tool + for _, t := range tools { + if t.Name == "pwsh" { + tool = &t + break + } + } + + if tool == nil { + return fmt.Errorf("unable to find powershell definition") } if version == "" { - v, err := get.FindGitHubRelease("PowerShell", "PowerShell") + v, err := get.FindGitHubRelease(tool.Owner, tool.Repo) if err != nil { return err } @@ -67,14 +70,8 @@ func MakeInstallPowershell() *cobra.Command { version = "v" + version } - fmt.Printf("Installing version: %s for: %s\n", version, dlArch) - semVer := semver.MustParse(version) majorVersion := semVer.Major() - // semVer := strings.TrimPrefix(version, "v") - - // majorVerDemlimiter := strings.Index(semVer, ".") - // majorVersion := semVer[:majorVerDemlimiter] installPath = fmt.Sprintf("%s/%d", installPath, majorVersion) @@ -82,35 +79,11 @@ func MakeInstallPowershell() *cobra.Command { installPath = strings.ReplaceAll(installPath, "$HOME", os.Getenv("HOME")) - if err := os.MkdirAll(installPath, 0755); err != nil && !os.IsExist(err) { - fmt.Printf("Error creating directory %s, error: %s\n", installPath, err.Error()) - } - - filename := fmt.Sprintf("powershell-%s-linux-%s.tar.gz", semVer, dlArch) - dlURL := fmt.Sprintf(githubDownloadTemplate, "PowerShell", "PowerShell", version, filename) - - fmt.Printf("Downloading from: %s\n", dlURL) - progress, _ := cmd.Flags().GetBool("progress") - outPath, err := get.DownloadFileP(dlURL, progress) + err := get.DownloadNested(tool, arch, osVer, version, installPath, progress, !progress) if err != nil { return err } - defer os.Remove(outPath) - - fmt.Printf("Downloaded to: %s\n", outPath) - - f, err := os.OpenFile(outPath, os.O_RDONLY, 0644) - if err != nil { - return err - } - defer f.Close() - - fmt.Printf("Unpacking Powershell to: %s\n", installPath) - - if err := archive.Untar(f, installPath, true, true); err != nil { - return err - } lnPath := "/usr/bin/pwsh" fmt.Printf("Creating Symbolic link to: %s\n", lnPath) diff --git a/pkg/get/download.go b/pkg/get/download.go index b3538de78..be5d6eb61 100644 --- a/pkg/get/download.go +++ b/pkg/get/download.go @@ -15,6 +15,7 @@ import ( "github.com/alexellis/arkade/pkg/config" "github.com/alexellis/arkade/pkg/env" "github.com/cheggaaa/pb/v3" + cp "github.com/otiai10/copy" ) const ( @@ -29,6 +30,59 @@ func (e *ErrNotFound) Error() string { return "server returned status: 404" } +func DownloadNested(tool *Tool, arch, operatingSystem, version string, movePath string, displayProgress, quiet bool) error { + downloadURL, err := GetDownloadURL(tool, + strings.ToLower(operatingSystem), + strings.ToLower(arch), + version, quiet) + if err != nil { + return err + } + + if !quiet { + fmt.Printf("Downloading: %s\n", downloadURL) + } + + outPath, err := DownloadFileP(downloadURL, displayProgress) + if err != nil { + return err + } + defer os.Remove(outPath) + + fmt.Printf("Downloaded to: %s\n", outPath) + + f, err := os.OpenFile(outPath, os.O_RDONLY, 0644) + if err != nil { + return err + } + defer f.Close() + + tempUnpackPath, err := os.MkdirTemp(os.TempDir(), "arkade-*") + if err != nil { + return err + } + defer os.RemoveAll(tempUnpackPath) + tempUnpackPath = fmt.Sprintf("%s/%s", tempUnpackPath, tool.Name) + + fmt.Printf("Unpacking %s to: %s\n", tool.Name, tempUnpackPath) + if err = archive.UntarNested(f, tempUnpackPath, true, true); err != nil { + return err + } + + if err := os.MkdirAll(movePath, 0755); err != nil && !os.IsExist(err) { + fmt.Printf("Error creating directory %s, error: %s\n", movePath, err.Error()) + } + + fmt.Printf("Copying binaries to: %s\n", movePath) + opts := cp.Options{ + AddPermission: 0755, + } + if err := cp.Copy(tempUnpackPath, movePath, opts); err != nil { + return err + } + return nil +} + func Download(tool *Tool, arch, operatingSystem, version string, movePath string, displayProgress, quiet bool) (string, string, error) { downloadURL, err := GetDownloadURL(tool, diff --git a/pkg/get/get.go b/pkg/get/get.go index 16122dc8b..f2dd15d61 100644 --- a/pkg/get/get.go +++ b/pkg/get/get.go @@ -9,6 +9,7 @@ import ( "log" "net" "net/http" + "regexp" "strings" "text/template" "time" @@ -19,6 +20,7 @@ import ( const GitHubVersionStrategy = "github" const k8sVersionStrategy = "k8s" +const goVersionStrategy = "go" var supportedOS = [...]string{"linux", "darwin", "ming"} var supportedArchitectures = [...]string{"x86_64", "arm", "amd64", "armv6l", "armv7l", "arm64", "aarch64"} @@ -64,6 +66,9 @@ type Tool struct { // NoExtension is required for tooling such as kubectx // which at time of writing is a bash script. NoExtension bool + + // true if tool should be used as system install only + SystemOnly bool } type ToolLocal struct { @@ -73,6 +78,7 @@ type ToolLocal struct { var templateFuncs = map[string]interface{}{ "HasPrefix": func(s, prefix string) bool { return strings.HasPrefix(s, prefix) }, + "ToLower": func(s string) string { return strings.ToLower(s) }, } func (tool Tool) IsArchive(quiet bool) (bool, error) { @@ -167,6 +173,14 @@ func (tool Tool) GetURL(os, arch, version string, quiet bool) (string, error) { version = v } + if tool.VersionStrategy == goVersionStrategy { + v, err := FindGoRelease() + if err != nil { + return "", err + } + version = v + } + if !quiet { log.Printf("Found: %s", version) } @@ -210,7 +224,6 @@ func getURLByGithubTemplate(tool Tool, os, arch, version string) (string, error) func FindGitHubRelease(owner, repo string) (string, error) { url := fmt.Sprintf("https://github.com/%s/%s/releases/latest", owner, repo) - client := makeHTTPClient(&githubTimeout, false) client.CheckRedirect = func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse @@ -280,6 +293,44 @@ func FindK8sRelease() (string, error) { return version, nil } +func FindGoRelease() (string, error) { + req, err := http.NewRequest(http.MethodGet, "https://go.dev/VERSION?m=text", nil) + if err != nil { + return "", err + } + + req.Header.Set("User-Agent", pkg.UserAgent()) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + + if res.Body == nil { + return "", fmt.Errorf("unexpected empty body") + } + + defer res.Body.Close() + body, _ := io.ReadAll(res.Body) + + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %d", res.StatusCode) + } + + content := strings.TrimSpace(string(body)) + exp, err := regexp.Compile(`^go(\d+.\d+.\d+)`) + if err != nil { + return "", err + } + + version := exp.FindStringSubmatch(content) + if len(version) < 2 { + return "", fmt.Errorf("failed to fetch go latest version number") + } + + return version[1], nil +} + func getBinaryURL(owner, repo, version, downloadName string) string { if in := strings.Index(downloadName, "/"); in > -1 { return fmt.Sprintf( diff --git a/pkg/get/get_test.go b/pkg/get/get_test.go index 85d0b7ccb..b92762227 100644 --- a/pkg/get/get_test.go +++ b/pkg/get/get_test.go @@ -7607,3 +7607,169 @@ func Test_DownloadLazyDocker(t *testing.T) { } } + +func Test_DownloadGo(t *testing.T) { + tools := MakeTools() + name := "go" + + tool := getTool(name, tools) + + const toolVersion = "1.22.4" + + tests := []test{ + { + os: "linux", + arch: arch64bit, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.linux-amd64.tar.gz", + }, + { + os: "linux", + arch: archARM64, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.linux-arm64.tar.gz", + }, + { + os: "linux", + arch: "armv6l", + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.linux-armv6l.tar.gz", + }, + { + os: "linux", + arch: archARM7, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.linux-armv6l.tar.gz", + }, + { + os: "darwin", + arch: arch64bit, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.darwin-amd64.tar.gz", + }, + { + os: "darwin", + arch: archDarwinARM64, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.darwin-arm64.tar.gz", + }, + { + os: "darwin", + arch: "armv6l", + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.darwin-armv6l.tar.gz", + }, + { + os: "darwin", + arch: archARM7, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.darwin-armv6l.tar.gz", + }, + { + os: "ming", + arch: archARM64, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.windows-arm64.zip", + }, + { + os: "ming", + arch: arch64bit, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.windows-amd64.zip", + }, + { + os: "ming", + arch: "armv6l", + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.windows-armv6l.zip", + }, + { + os: "ming", + arch: archARM7, + version: toolVersion, + url: "https://go.dev/dl/go1.22.4.windows-armv6l.zip", + }, + } + + for _, tc := range tests { + got, err := tool.GetURL(tc.os, tc.arch, tc.version, false) + if err != nil { + t.Fatal(err) + } + if got != tc.url { + t.Errorf("\nwant: %s, \n got: %s", tc.url, got) + } + } + +} + +func Test_DownloadPowershell(t *testing.T) { + tools := MakeTools() + name := "pwsh" + + tool := getTool(name, tools) + + const toolVersion = "v7.4.3" + + tests := []test{ + { + os: "linux", + arch: arch64bit, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-linux-x64.tar.gz", + }, + { + os: "linux", + arch: archARM64, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-linux-arm64.tar.gz", + }, + { + os: "linux", + arch: "armv6l", + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-linux-arm32.tar.gz", + }, + { + os: "linux", + arch: archARM7, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-linux-arm32.tar.gz", + }, + { + os: "darwin", + arch: arch64bit, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-osx-x64.tar.gz", + }, + { + os: "darwin", + arch: archDarwinARM64, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/powershell-7.4.3-osx-arm64.tar.gz", + }, + { + os: "ming", + arch: archARM64, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/PowerShell-7.4.3-win-arm64.zip", + }, + { + os: "ming", + arch: arch64bit, + version: toolVersion, + url: "https://github.com/PowerShell/PowerShell/releases/download/v7.4.3/PowerShell-7.4.3-win-x64.zip", + }, + } + + for _, tc := range tests { + got, err := tool.GetURL(tc.os, tc.arch, tc.version, false) + if err != nil { + t.Fatal(err) + } + if got != tc.url { + t.Errorf("\nwant: %s, \n got: %s", tc.url, got) + } + } + +} diff --git a/pkg/get/tools.go b/pkg/get/tools.go index 53d07400a..cf811003b 100644 --- a/pkg/get/tools.go +++ b/pkg/get/tools.go @@ -21,6 +21,17 @@ func (t Tools) Less(i, j int) bool { type Tools []Tool +func MakeToolsWithoutSystemApp() Tools { + allTools := MakeTools() + tools := []Tool{} + for _, tool := range allTools { + if !tool.SystemOnly { + tools = append(tools, tool) + } + } + return tools +} + func MakeTools() Tools { tools := []Tool{} @@ -4183,5 +4194,67 @@ https://github.com/{{.Owner}}/{{.Repo}}/releases/download/{{.Version}}/{{.Name}} {{.Name}}_{{.VersionNumber}}_{{$osStr}}_{{$arch}}.{{$ext}}`, }) + + tools = append(tools, + Tool{ + Owner: "go", + Repo: "go", + Name: "go", + SystemOnly: true, + VersionStrategy: goVersionStrategy, + Description: "Build simple, secure, scalable systems with Go", + URLTemplate: ` + {{$os := .OS}} + {{$arch := .Arch}} + {{$ext := "tar.gz"}} + + {{- if (or (eq .Arch "aarch64") (eq .Arch "arm64")) -}} + {{$arch = "arm64"}} + {{- else if eq .Arch "x86_64" -}} + {{ $arch = "amd64" }} + {{- else if eq .Arch "armv7l" -}} + {{ $arch = "armv6l" }} + {{- end -}} + + {{ if HasPrefix .OS "ming" -}} + {{$os = "windows"}} + {{$ext = "zip"}} + {{- end -}} + + https://{{.Name}}.dev/dl/{{.Name}}{{.VersionNumber}}.{{$os}}-{{$arch}}.{{$ext}}`, + }) + + tools = append(tools, + Tool{ + Owner: "PowerShell", + Repo: "PowerShell", + Name: "pwsh", + VersionStrategy: GitHubVersionStrategy, + SystemOnly: true, + Description: "PowerShell is a cross-platform automation and configuration tool/framework", + URLTemplate: ` + {{$os := .OS}} + {{$arch := .Arch}} + {{$ext := "tar.gz"}} + {{$zipPrefix := ToLower .Repo}} + + {{- if (or (eq .Arch "aarch64") (eq .Arch "arm64")) -}} + {{$arch = "arm64"}} + {{- else if eq .Arch "x86_64" -}} + {{ $arch = "x64" }} + {{- else if (or (eq .Arch "armv6l") (eq .Arch "armv7l")) -}} + {{ $arch = "arm32" }} + {{- end -}} + + {{- if eq .OS "darwin" -}} + {{$os = "osx"}} + {{- else if HasPrefix .OS "ming" -}} + {{$os = "win"}} + {{$ext = "zip"}} + {{$zipPrefix = .Repo}} + {{- end -}} + + https://github.com/{{.Owner}}/{{.Repo}}/releases/download/{{.Version}}/{{$zipPrefix}}-{{.VersionNumber}}-{{$os}}-{{$arch}}.{{$ext}}`, + }) return tools } diff --git a/pkg/get/url_checker_test.go b/pkg/get/url_checker_test.go index 7679c4235..628e97e20 100644 --- a/pkg/get/url_checker_test.go +++ b/pkg/get/url_checker_test.go @@ -14,6 +14,8 @@ func Test_CheckTools(t *testing.T) { tools := MakeTools() toolsToSkip := []string{ "kumactl", // S3 bucket disallow HEAD requests + // https://github.com/vladimirvivien/ktop/issues/46 + "ktop", // latest release does not include binary } os := "linux"