Refactor/sync data

cmd/commands/datasetIngestor.go

@@ -355,7 +355,7 @@ For Windows you need instead to specify -user username:password on the command l
 						log.Printf("Attachment file %v added to dataset  %v\n", addAttachment, datasetId)
 					}
 					if copyFlag {
-						err := datasetIngestor.SyncDataToFileserver(datasetId, user, RSYNCServer, datasetSourceFolder, absFileListing)
+						err := datasetIngestor.SyncLocalDataToFileserver(datasetId, user, RSYNCServer, datasetSourceFolder, absFileListing, os.Stdout)
 						if err == nil {
 							// delayed enabling
 							archivable = true

datasetIngestor/syncDataToFileserver_unix.go

@@ -1,39 +1,41 @@
+//go:build aix || darwin || dragonfly || freebsd || (js && wasm) || linux || nacl || netbsd || openbsd || solaris
 // +build aix darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris
 
 // very important: there must be an empty line after the build flag line .
 package datasetIngestor
 
 import (
 	"fmt"
-	"log"
-	"os"
+	"io"
 	"os/exec"
+	"regexp"
 	"strings"
+
 	version "github.com/mcuadros/go-version"
-	"regexp"
 )
 
 // functionality needed for "de-central" data
-func SyncDataToFileserver(datasetId string, user map[string]string, RSYNCServer string, sourceFolder string, absFileListing string) (err error) {
+// copies data from a local machine to a fileserver, uses RSync underneath
+func SyncLocalDataToFileserver(datasetId string, user map[string]string, RSYNCServer string, sourceFolder string, absFileListing string, cmdOutput io.Writer) (err error) {
 	username := user["username"]
 	shortDatasetId := strings.Split(datasetId, "/")[1]
-	log.Println("short dataset id:", shortDatasetId)
 	destFolder := "archive/" + shortDatasetId + sourceFolder
 	serverConnectString := fmt.Sprintf("%s@%s:%s", username, RSYNCServer, destFolder)
 	// append trailing slash to sourceFolder to indicate that the *contents* of the folder should be copied
 	// no special handling for blanks in sourceFolder needed here
 	fullSourceFolderPath := sourceFolder + "/"
-	
+
 	versionNumber, err := getRsyncVersion()
 	if err != nil {
-		log.Fatal("Error getting rsync version: ", err)
+		return fmt.Errorf("error getting rsync version: %v", err)
 	}
-	
+
 	rsyncCmd := buildRsyncCmd(versionNumber, absFileListing, fullSourceFolderPath, serverConnectString)
-
-	// Show rsync's output	
-	rsyncCmd.Stderr = os.Stderr
-	log.Printf("Running: %v.\n", rsyncCmd.Args)
+
+	// Show rsync's output
+	rsyncCmd.Stdout = cmdOutput
+	rsyncCmd.Stderr = cmdOutput
+	fmt.Fprintf(cmdOutput, "Running: %v.\n", rsyncCmd.Args)
 	err = rsyncCmd.Run()
 	return err
 }
@@ -46,15 +48,15 @@ func getRsyncVersion() (string, error) {
 		return "", err
 	}
 	version := string(output)
-	
+
 	// Use a regular expression to find the version number.
 	// It will match the first occurrence of a string in the format "x.y.z" in the `version` string, where "x", "y", and "z" are one or more digits.
 	re := regexp.MustCompile(`\d+\.\d+\.\d+`)
 	versionNumber := re.FindString(version)
 	if versionNumber == "" {
 		return "", fmt.Errorf("could not find version number in rsync version string: %s", version)
 	}
-	
+
 	return versionNumber, nil
 }
 

datasetIngestor/syncDataToFileserver_windows.go

@@ -2,14 +2,15 @@
 package datasetIngestor
 
 import (
-	"log"
+	"fmt"
+	"io"
 	"path"
 	"regexp"
 	"strings"
 )
 
-func SyncDataToFileserver(datasetId string, user map[string]string, RSYNCServer string, sourceFolder string, absFileListing string) (err error) {
-
+// copies data from a local machine to a fileserver, uses scp underneath
+func SyncLocalDataToFileserver(datasetId string, user map[string]string, RSYNCServer string, sourceFolder string, absFileListing string, commandOutput io.Writer) (err error) {
 	username := user["username"]
 	password := user["password"]
 	shortDatasetId := strings.Split(datasetId, "/")[1]
@@ -26,9 +27,6 @@ func SyncDataToFileserver(datasetId string, user map[string]string, RSYNCServer
 	destFolder := "archive/" + shortDatasetId + strings.Join(destparts[0:len(destparts)-1], "/")
 	destFolder2 := "archive/" + shortDatasetId + strings.Join(destparts[0:len(destparts)], "/")
 
-	// fmt.Println("Destination folder:", destFolder)
-	// fmt.Println("Sourcefolder:", sourceFolder)
-
 	// add port number if missing
 	FullRSYNCServer := RSYNCServer
 	if !strings.Contains(RSYNCServer, ":") {
@@ -51,16 +49,19 @@ func SyncDataToFileserver(datasetId string, user map[string]string, RSYNCServer
 	if absFileListing != "" {
 		lines, err := readLines(absFileListing)
 		if err != nil {
-			log.Fatalf("Could not read filellist, readLines: %s", err)
+			return fmt.Errorf("could not read filelist, readlines: %v", err)
 		}
 		for _, line := range lines {
 			windowsSource := re.ReplaceAllString(path.Join(sourceFolder, line), "$1:/")
-			// log.Printf("Copying data via scp from %s to %s\n", windowsSource, destFolder2)
+			fmt.Fprintf(commandOutput, "Copying data via scp from %s to %s\n", windowsSource, destFolder)
 			err = c.Send(destFolder2, windowsSource)
+			if err != nil {
+				return err
+			}
 		}
 	} else {
 		windowsSource := re.ReplaceAllString(sourceFolder, "$1:/")
-		// log.Printf("Copying data via scp from %s to %s\n", windowsSource, destFolder)
+		fmt.Fprintf(commandOutput, "Copying data via scp from %s to %s\n", windowsSource, destFolder)
 		err = c.Send(destFolder, windowsSource)
 	}
 	return err