Merge branch 'improve-downloads' into master

CHANGELOG.md

@@ -1,3 +1,12 @@
+## 1.61.0 17-May-2021
+
+## NEW FEATURES
+
+* Command "downloads get" can also use direct URL instead of tarball names
+* Add (hidden) command "downloads add-stdin"
+* Add command "downloads add-remote"
+* Add option `--version` to "downloads list"
+
 ## 1.60.0 01-May-2021
 
 ## NEW FEATURES
@@ -7,7 +16,7 @@
 
 ## BUGS FIXED
 
-* (@ogroveln) [Allow other custom mysqld than mysqld-debug](https://github.com/datacharmer/dbdeployer/pull/135)
+* (@ogrovlen) [Allow other custom mysqld than mysqld-debug](https://github.com/datacharmer/dbdeployer/pull/135)
 
 ## 1.59.0 27-Feb-2021
 

cmd/downloads.go

@@ -16,10 +16,12 @@
 package cmd
 
 import (
+ "bufio"
  "encoding/json"
  "fmt"
  "os"
  "path"
+ "path/filepath"
  "regexp"
  "runtime"
  "strings"
@@ -49,6 +51,7 @@ func listRemoteTarballs(cmd *cobra.Command, args []string) {
 
  flavor, _ := cmd.Flags().GetString(globals.FlavorLabel)
  OS, _ := cmd.Flags().GetString(globals.OSLabel)
+ version, _ := cmd.Flags().GetString(globals.VersionLabel)
  OS = strings.ToLower(OS)
  flavor = strings.ToLower(flavor)
  showUrl, _ := cmd.Flags().GetBool(globals.ShowUrlLabel)
@@ -96,11 +99,16 @@ func listRemoteTarballs(cmd *cobra.Command, args []string) {
  cells = append(cells, &simpletable.Cell{Text: tb.Flavor})
  cells = append(cells, &simpletable.Cell{Align: simpletable.AlignRight, Text: humanize.Bytes(uint64(tb.Size))})
  cells = append(cells, &simpletable.Cell{Text: minimalTag})
- if flavor == strings.ToLower(tb.Flavor) || flavor == "" || flavor == "all" {
- if OS == strings.ToLower(tb.OperatingSystem) || OS == "" || OS == "all" {
- table.Body.Cells = append(table.Body.Cells, cells)
- }
+ if version != "" && version != "all" && version != tb.Version && version != tb.ShortVersion {
+ continue
+ }
+ if flavor != "" && flavor != "all" && flavor != strings.ToLower(tb.Flavor) {
+ continue
+ }
+ if OS != "" && strings.ToLower(OS) != "all" && OS != strings.ToLower(tb.OperatingSystem) {
+ continue
  }
+ table.Body.Cells = append(table.Body.Cells, cells)
  }
  table.SetStyle(simpletable.StyleCompactLite)
  table.Println()
@@ -116,14 +124,52 @@ func getRemoteTarball(cmd *cobra.Command, args []string) {
  quiet, _ := cmd.Flags().GetBool(globals.QuietLabel)
  progressStep, _ := cmd.Flags().GetInt64(globals.ProgressStepLabel)
  dryRun, _ := cmd.Flags().GetBool(globals.DryRunLabel)
+ wantedOs, _ := cmd.Flags().GetString(globals.OSLabel)
 
  var tarball downloads.TarballDescription
  var err error
  var fileName string
 
- tarball, err = downloads.FindTarballByName(args[0])
- if err != nil {
- common.Exitf(1, "%s", err)
+ wanted := args[0]
+
+ if common.IsUrl(wanted) {
+ storedTarball, err := downloads.FindTarballByUrl(wanted)
+ if err == nil {
+ tarball = storedTarball
+ } else {
+ name := filepath.Base(wanted)
+ lowerName := strings.ToLower(name)
+ flavor, version, shortVersion, err := common.FindTarballInfo(name)
+ if err != nil {
+ common.Exitf(1, "%s", err)
+ }
+ tarballOs := runtime.GOOS
+ if wantedOs != "" {
+ tarballOs = wantedOs
+ } else {
+ switch {
+ case strings.Contains(lowerName, "linux"):
+ tarballOs = "linux"
+ case strings.Contains(lowerName, "macos") || strings.Contains(lowerName, "darwin"):
+ tarballOs = "darwin"
+ default:
+ common.Exitf(1, "unable to determine the operating system of tarball '%s'", name)
+ }
+ }
+ tarball = downloads.TarballDescription{
+ Name: name,
+ Url: wanted,
+ Flavor: flavor,
+ ShortVersion: shortVersion,
+ Version: version,
+ OperatingSystem: tarballOs,
+ }
+ }
+ } else {
+ tarball, err = downloads.FindTarballByName(wanted)
+ if err != nil {
+ common.Exitf(1, "%s", err)
+ }
  }
 
  fileName = tarball.Name
@@ -204,8 +250,9 @@ func getRemoteTarballByVersion(cmd *cobra.Command, args []string) {
 
  tarball, err = downloads.FindOrGuessTarballByVersionFlavorOS(version, flavor, OS, minimal, newest, guessLatest)
  if err != nil {
- common.Exitf(1, "Error getting version %s (%s-%s)[minimal: %v - newest: %v - guess: %v]: %s",
- version, flavor, OS, minimal, newest, guessLatest, err)
+ common.Exit(1, fmt.Sprintf("Error getting version %s (%s-%s)[minimal: %v - newest: %v - guess: %v]: %s",
+ version, flavor, OS, minimal, newest, guessLatest, err),
+ fmt.Sprintf("Try using 'dbdeployer downloads get-remote mysql %s %s'", version, OS))
  }
 
  fileName = tarball.Name
@@ -239,6 +286,7 @@ func getUnpackRemoteTarball(cmd *cobra.Command, args []string) {
  deleteAfterUnpack, _ := cmd.Flags().GetBool(globals.DeleteAfterUnpackLabel)
  getRemoteTarball(cmd, args)
 
+ // unpack flags are passed from current command and resolved in the called function
  unpackTarball(cmd, args)
  if deleteAfterUnpack {
  if downloadedTarball == "" {
@@ -365,6 +413,61 @@ func importTarballCollection(cmd *cobra.Command, args []string) {
  fmt.Printf("Tarball list imported from %s to %s\n", fileName, downloads.TarballFileRegistry)
 }
 
+func addRemoteTarballToCollection(cmd *cobra.Command, args []string) {
+ flags := cmd.Flags()
+ if len(args) < 3 {
+ common.Exit(1, "command 'add' requires tarball type, short version, and operating system")
+ }
+ tarballType := downloads.TarballType(args[0])
+
+ list, err := downloads.GetRemoteTarballList(tarballType, args[1], args[2], true)
+
+ if err != nil {
+ common.Exitf(1, "error getting remote tarball description: %s", err)
+ }
+
+ var tarballCollection = downloads.DefaultTarballRegistry
+ minimal, _ := flags.GetBool(globals.MinimalLabel)
+ overwrite, _ := flags.GetBool(globals.OverwriteLabel)
+ var added []downloads.TarballDescription
+ for _, tb := range list {
+ if minimal && !tb.Minimal {
+ continue
+ }
+ existingTarball, err := downloads.FindTarballByName(tb.Name)
+ if err == nil {
+ if overwrite {
+ var newList []downloads.TarballDescription
+ newList, err = downloads.DeleteTarball(tb.Name)
+ if err != nil {
+ common.Exitf(1, "error removing tarball %s from list", tb.Name)
+ }
+ tarballCollection.Tarballs = newList
+ } else {
+ displayTarball(existingTarball)
+ fmt.Println()
+ common.Exitf(1, "tarball %s already in the list", tb.Name)
+ }
+ }
+
+ tb.DateAdded = time.Now().Format("2006-01-02 15:04")
+ tb.Notes = fmt.Sprintf("added with version %s", common.VersionDef)
+ tarballCollection.Tarballs = append(tarballCollection.Tarballs, tb)
+ added = append(added, tb)
+ }
+ if len(added) == 0 {
+ common.Exitf(1, "no tarballs found to add")
+ }
+ err = downloads.WriteTarballFileInfo(tarballCollection)
+ if err != nil {
+ common.Exitf(1, "error writing tarball list: %s", err)
+ }
+ fmt.Printf("Tarball below added to %s\n", downloads.TarballFileRegistry)
+ for _, tb := range added {
+ displayTarball(tb)
+ }
+}
+
 func addTarballToCollection(cmd *cobra.Command, args []string) {
  flags := cmd.Flags()
  if len(args) < 1 {
@@ -431,6 +534,47 @@ func addTarballToCollection(cmd *cobra.Command, args []string) {
  displayTarball(tarballDesc)
 }
 
+func addTarballToCollectionFromStdin(cmd *cobra.Command, args []string) {
+ var err error
+ var tarballCollection = downloads.DefaultTarballRegistry
+ overwrite, _ := cmd.Flags().GetBool(globals.OverwriteLabel)
+ scanner := bufio.NewScanner(os.Stdin)
+
+ text := ""
+ for scanner.Scan() {
+ text += scanner.Text()
+ }
+ var tarballDesc downloads.TarballDescription
+ err = json.Unmarshal([]byte(text), &tarballDesc)
+ if err != nil {
+ common.Exitf(1, "error decoding JSON item:%s", err)
+ }
+ existingTarball, err := downloads.FindTarballByName(tarballDesc.Name)
+ if err == nil {
+ if overwrite {
+ var newList []downloads.TarballDescription
+ newList, err = downloads.DeleteTarball(tarballDesc.Name)
+ if err != nil {
+ common.Exitf(1, "error removing tarball %s from list", tarballDesc.Name)
+ }
+ tarballCollection.Tarballs = newList
+ } else {
+ displayTarball(existingTarball)
+ fmt.Println()
+ common.Exitf(1, "tarball %s already in the list", tarballDesc.Name)
+ }
+ }
+ tarballDesc.Notes = fmt.Sprintf("added with version %s", common.VersionDef)
+ tarballCollection.Tarballs = append(tarballCollection.Tarballs, tarballDesc)
+
+ err = downloads.WriteTarballFileInfo(tarballCollection)
+ if err != nil {
+ common.Exitf(1, "error writing tarball list: %s", err)
+ }
+ fmt.Printf("Tarball below added to %s\n", downloads.TarballFileRegistry)
+ displayTarball(tarballDesc)
+}
+
 var downloadsListCmd = &cobra.Command{
  Use: "list [options]",
  Aliases: []string{"index"},
@@ -523,6 +667,24 @@ var downloadsAddCmd = &cobra.Command{
  Run: addTarballToCollection,
 }
 
+var downloadsAddRemoteCmd = &cobra.Command{
+ Use: "add-remote tarball-type short-version operating-system",
+ Short: "Adds a tarball to the list, by searching MySQL downloads site ",
+ Long: `This command can add a tarball by searching the MySQL site for one of these
+tarball types: mysql, cluster, shell`,
+
+ Run: addRemoteTarballToCollection,
+}
+
+var downloadsAddStdinCmd = &cobra.Command{
+ Use: "add-stdin ",
+ Short: "Adds a tarball to the list from standard input",
+ Long: ``,
+ Hidden: true,
+
+ Run: addTarballToCollectionFromStdin,
+}
+
 var downloadsCmd = &cobra.Command{
  Use: "downloads",
  Short: "Manages remote tarballs",
@@ -540,10 +702,13 @@ func init() {
  downloadsCmd.AddCommand(downloadsGetByVersionCmd)
  downloadsCmd.AddCommand(downloadsGetUnpackCmd)
  downloadsCmd.AddCommand(downloadsAddCmd)
+ downloadsCmd.AddCommand(downloadsAddRemoteCmd)
+ downloadsCmd.AddCommand(downloadsAddStdinCmd)
 
  downloadsListCmd.Flags().BoolP(globals.ShowUrlLabel, "", false, "Show the URL")
  downloadsListCmd.Flags().String(globals.FlavorLabel, "", "Which flavor will be listed")
  downloadsListCmd.Flags().String(globals.OSLabel, "", "Which OS will be listed")
+ downloadsListCmd.Flags().String(globals.VersionLabel, "", "Which version will be listed")
 
  downloadsGetByVersionCmd.Flags().BoolP(globals.NewestLabel, "", false, "Choose only the newest tarballs not yet downloaded")
  downloadsGetByVersionCmd.Flags().BoolP(globals.DryRunLabel, "", false, "Show what would be downloaded, but don't run it")
@@ -557,8 +722,10 @@ func init() {
  downloadsGetCmd.Flags().BoolP(globals.QuietLabel, "", false, "Do not show download progress")
  downloadsGetCmd.Flags().Int64P(globals.ProgressStepLabel, "", globals.ProgressStepValue, "Progress interval")
  downloadsGetCmd.Flags().BoolP(globals.DryRunLabel, "", false, "Show what would be downloaded, but don't run it")
+ downloadsGetCmd.Flags().String(globals.OSLabel, "", "Set the OS of the tarball")
 
  downloadsGetUnpackCmd.Flags().BoolP(globals.DeleteAfterUnpackLabel, "", false, "Delete the tarball after successful unpack")
+ downloadsGetUnpackCmd.Flags().Bool(globals.DryRunLabel, false, "Show Get operations, but do not run them")
 
  downloadsAddCmd.Flags().String(globals.OSLabel, "", "Define the tarball OS (default: current OS)")
  downloadsAddCmd.Flags().String(globals.FlavorLabel, "", "Define the tarball flavor")
@@ -570,6 +737,11 @@ func init() {
  _ = downloadsAddCmd.MarkFlagRequired(globals.UrlLabel)
  _ = downloadsAddCmd.MarkFlagRequired(globals.OSLabel)
 
+ downloadsAddRemoteCmd.Flags().BoolP(globals.OverwriteLabel, "", false, "Overwrite existing entry")
+ downloadsAddRemoteCmd.Flags().BoolP(globals.MinimalLabel, "", false, "Define whether the wanted tarball is a minimal one")
+
+ downloadsAddStdinCmd.Flags().BoolP(globals.OverwriteLabel, "", false, "Overwrite existing entry")
+
  // downloadsGetUnpack needs the same flags that cmdUnpack has
  downloadsGetUnpackCmd.Flags().Int64P(globals.ProgressStepLabel, "", globals.ProgressStepValue, "Progress interval")
  downloadsGetUnpackCmd.PersistentFlags().Int(globals.VerbosityLabel, 1, "Level of verbosity during unpack (0=none, 2=maximum)")

cmd/unpack.go

@@ -58,6 +58,7 @@ func unpackTarball(cmd *cobra.Command, args []string) {
 
  overwrite, _ := flags.GetBool(globals.OverwriteLabel)
  flavor, _ := flags.GetString(globals.FlavorLabel)
+ dryRun, _ := flags.GetBool(globals.DryRunLabel)
  if flavor == "" {
  baseName := common.BaseName(tarball)
  flavor = common.DetectTarballFlavor(baseName)
@@ -98,12 +99,16 @@ func unpackTarball(cmd *cobra.Command, args []string) {
  }
  if common.DirExists(destination) && !isShell {
  if overwrite {
- isDeleted, err := deleteBinaries(Basedir, Prefix+Version, false)
- if !isDeleted {
- common.Exitf(1, "directory %s could not be removed", Prefix+Version)
- }
- if err != nil {
- common.Exitf(1, "error removing directory %s: %s", Prefix+Version, err)
+ if dryRun {
+ fmt.Printf("delete binaries %s %s\n", Basedir, Prefix+Version)
+ } else {
+ isDeleted, err := deleteBinaries(Basedir, Prefix+Version, false)
+ if !isDeleted {
+ common.Exitf(1, "directory %s could not be removed", Prefix+Version)
+ }
+ if err != nil {
+ common.Exitf(1, "error removing directory %s: %s", Prefix+Version, err)
+ }
  }
  } else {
  common.Exitf(1, globals.ErrNamedDirectoryAlreadyExists, "destination directory", destination)
@@ -132,12 +137,17 @@ func unpackTarball(cmd *cobra.Command, args []string) {
  bareName = extracted[0 : len(extracted)-len(globals.TarGzExt)]
  if isShell {
  common.CondPrintf("Merging shell tarball %s to %s\n", common.ReplaceLiteralHome(tarball), common.ReplaceLiteralHome(destination))
- err := unpack.MergeShell(tarball, foundExtension, Basedir, destination, bareName, verbosity)
- common.ErrCheckExitf(err, 1, "error while unpacking mysql shell tarball : %s", err)
+ if !dryRun {
+ err := unpack.MergeShell(tarball, foundExtension, Basedir, destination, bareName, verbosity)
+ common.ErrCheckExitf(err, 1, "error while unpacking mysql shell tarball : %s", err)
+ }
  return
  }
 
  common.CondPrintf("Unpacking tarball %s to %s\n", tarball, common.ReplaceLiteralHome(destination))
+ if dryRun {
+ return
+ }
  //verbosity_level := unpack.VERBOSE
  // err := unpack.UnpackTar(tarball, Basedir, verbosity)
  err = extractFunc(tarball, Basedir, verbosity)
@@ -192,6 +202,7 @@ func init() {
  unpackCmd.PersistentFlags().String(globals.PrefixLabel, "", "Prefix for the final expanded directory")
  unpackCmd.PersistentFlags().Bool(globals.ShellLabel, false, "Unpack a shell tarball into the corresponding server directory")
  unpackCmd.PersistentFlags().Bool(globals.OverwriteLabel, false, "Overwrite the destination directory if already exists")
+ unpackCmd.PersistentFlags().Bool(globals.DryRunLabel, false, "Show unpack operations, but do not run them")
  unpackCmd.PersistentFlags().String(globals.TargetServerLabel, "", "Uses a different server to unpack a shell tarball")
  unpackCmd.PersistentFlags().String(globals.FlavorLabel, "", "Defines the tarball flavor (MySQL, NDB, Percona Server, etc)")
 }

common/VERSION

@@ -1 +1 @@
-1.60.0
+1.61.0

defaults/defaults.go

@@ -137,7 +137,7 @@ var (
  PxcPrefix: "pxc_msb_",
  DefaultSandboxExecutable: "default",
  DownloadNameLinux: "mysql-{{.Version}}-linux-glibc2.17-x86_64{{.Minimal}}.{{.Ext}}",
- DownloadNameMacOs: "mysql-{{.Version}}-macos10.15-x86_64.{{.Ext}}",
+ DownloadNameMacOs: "mysql-{{.Version}}-macos11-x86_64.{{.Ext}}",
  DownloadUrl: "https://dev.mysql.com/get/Downloads/MySQL",
  Timestamp: time.Now().Format(time.UnixDate),
  }