diff --git a/cmd/commands/datasetIngestor.go b/cmd/commands/datasetIngestor.go index 139fb64..d33291d 100644 --- a/cmd/commands/datasetIngestor.go +++ b/cmd/commands/datasetIngestor.go @@ -355,7 +355,7 @@ For Windows you need instead to specify -user username:password on the command l log.Printf("Attachment file %v added to dataset %v\n", addAttachment, datasetId) } if copyFlag { - err := datasetIngestor.SyncDataToFileserver(datasetId, user, RSYNCServer, datasetSourceFolder, absFileListing) + err := datasetIngestor.SyncLocalDataToFileserver(datasetId, user, RSYNCServer, datasetSourceFolder, absFileListing, os.Stdout) if err == nil { // delayed enabling archivable = true diff --git a/datasetIngestor/syncDataToFileserver_unix.go b/datasetIngestor/syncDataToFileserver_unix.go index 86f3ab0..397dc37 100644 --- a/datasetIngestor/syncDataToFileserver_unix.go +++ b/datasetIngestor/syncDataToFileserver_unix.go @@ -1,3 +1,4 @@ +//go:build aix || darwin || dragonfly || freebsd || (js && wasm) || linux || nacl || netbsd || openbsd || solaris // +build aix darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris // very important: there must be an empty line after the build flag line . @@ -5,35 +6,36 @@ package datasetIngestor import ( "fmt" - "log" - "os" + "io" "os/exec" + "regexp" "strings" + version "github.com/mcuadros/go-version" - "regexp" ) // functionality needed for "de-central" data -func SyncDataToFileserver(datasetId string, user map[string]string, RSYNCServer string, sourceFolder string, absFileListing string) (err error) { +// copies data from a local machine to a fileserver, uses RSync underneath +func SyncLocalDataToFileserver(datasetId string, user map[string]string, RSYNCServer string, sourceFolder string, absFileListing string, cmdOutput io.Writer) (err error) { username := user["username"] shortDatasetId := strings.Split(datasetId, "/")[1] - log.Println("short dataset id:", shortDatasetId) destFolder := "archive/" + shortDatasetId + sourceFolder serverConnectString := fmt.Sprintf("%s@%s:%s", username, RSYNCServer, destFolder) // append trailing slash to sourceFolder to indicate that the *contents* of the folder should be copied // no special handling for blanks in sourceFolder needed here fullSourceFolderPath := sourceFolder + "/" - + versionNumber, err := getRsyncVersion() if err != nil { - log.Fatal("Error getting rsync version: ", err) + return fmt.Errorf("error getting rsync version: %v", err) } - + rsyncCmd := buildRsyncCmd(versionNumber, absFileListing, fullSourceFolderPath, serverConnectString) - - // Show rsync's output - rsyncCmd.Stderr = os.Stderr - log.Printf("Running: %v.\n", rsyncCmd.Args) + + // Show rsync's output + rsyncCmd.Stdout = cmdOutput + rsyncCmd.Stderr = cmdOutput + fmt.Fprintf(cmdOutput, "Running: %v.\n", rsyncCmd.Args) err = rsyncCmd.Run() return err } @@ -46,7 +48,7 @@ func getRsyncVersion() (string, error) { return "", err } version := string(output) - + // Use a regular expression to find the version number. // It will match the first occurrence of a string in the format "x.y.z" in the `version` string, where "x", "y", and "z" are one or more digits. re := regexp.MustCompile(`\d+\.\d+\.\d+`) @@ -54,7 +56,7 @@ func getRsyncVersion() (string, error) { if versionNumber == "" { return "", fmt.Errorf("could not find version number in rsync version string: %s", version) } - + return versionNumber, nil } diff --git a/datasetIngestor/syncDataToFileserver_windows.go b/datasetIngestor/syncDataToFileserver_windows.go index 7cd77a2..a347dc7 100644 --- a/datasetIngestor/syncDataToFileserver_windows.go +++ b/datasetIngestor/syncDataToFileserver_windows.go @@ -2,14 +2,15 @@ package datasetIngestor import ( - "log" + "fmt" + "io" "path" "regexp" "strings" ) -func SyncDataToFileserver(datasetId string, user map[string]string, RSYNCServer string, sourceFolder string, absFileListing string) (err error) { - +// copies data from a local machine to a fileserver, uses scp underneath +func SyncLocalDataToFileserver(datasetId string, user map[string]string, RSYNCServer string, sourceFolder string, absFileListing string, commandOutput io.Writer) (err error) { username := user["username"] password := user["password"] shortDatasetId := strings.Split(datasetId, "/")[1] @@ -26,9 +27,6 @@ func SyncDataToFileserver(datasetId string, user map[string]string, RSYNCServer destFolder := "archive/" + shortDatasetId + strings.Join(destparts[0:len(destparts)-1], "/") destFolder2 := "archive/" + shortDatasetId + strings.Join(destparts[0:len(destparts)], "/") - // fmt.Println("Destination folder:", destFolder) - // fmt.Println("Sourcefolder:", sourceFolder) - // add port number if missing FullRSYNCServer := RSYNCServer if !strings.Contains(RSYNCServer, ":") { @@ -51,16 +49,19 @@ func SyncDataToFileserver(datasetId string, user map[string]string, RSYNCServer if absFileListing != "" { lines, err := readLines(absFileListing) if err != nil { - log.Fatalf("Could not read filellist, readLines: %s", err) + return fmt.Errorf("could not read filelist, readlines: %v", err) } for _, line := range lines { windowsSource := re.ReplaceAllString(path.Join(sourceFolder, line), "$1:/") - // log.Printf("Copying data via scp from %s to %s\n", windowsSource, destFolder2) + fmt.Fprintf(commandOutput, "Copying data via scp from %s to %s\n", windowsSource, destFolder) err = c.Send(destFolder2, windowsSource) + if err != nil { + return err + } } } else { windowsSource := re.ReplaceAllString(sourceFolder, "$1:/") - // log.Printf("Copying data via scp from %s to %s\n", windowsSource, destFolder) + fmt.Fprintf(commandOutput, "Copying data via scp from %s to %s\n", windowsSource, destFolder) err = c.Send(destFolder, windowsSource) } return err