From 7b05b5997c08aa5e0b67659a61bc4b086ada4cf4 Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Thu, 25 Jul 2024 08:11:22 -0700 Subject: [PATCH 01/16] WIP: implement transfer report --- cmd/flag/transfer_report.go | 33 ++++++++ cmd/subcmd/get.go | 97 +++++++++++++++++---- commons/transfer_report.go | 162 ++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- 5 files changed, 278 insertions(+), 20 deletions(-) create mode 100644 cmd/flag/transfer_report.go create mode 100644 commons/transfer_report.go diff --git a/cmd/flag/transfer_report.go b/cmd/flag/transfer_report.go new file mode 100644 index 0000000..30cb60d --- /dev/null +++ b/cmd/flag/transfer_report.go @@ -0,0 +1,33 @@ +package flag + +import ( + "github.com/spf13/cobra" +) + +type TransferReportFlagValues struct { + ReportPath string + Report bool + ReportToStdout bool +} + +var ( + transferReportFlagValues TransferReportFlagValues +) + +func SetTransferReportFlags(command *cobra.Command) { + command.Flags().StringVar(&transferReportFlagValues.ReportPath, "report", "", "Create a transfer report, give path to create a file, empty string or '-' will output to stdout") +} + +func GetTransferReportFlagValues(command *cobra.Command) *TransferReportFlagValues { + if command.Flags().Changed("report") { + transferReportFlagValues.Report = true + } + + transferReportFlagValues.ReportToStdout = false + + if transferReportFlagValues.ReportPath == "-" || len(transferReportFlagValues.ReportPath) == 0 { + transferReportFlagValues.ReportToStdout = true + } + + return &transferReportFlagValues +} diff --git a/cmd/subcmd/get.go b/cmd/subcmd/get.go index 1f1e3c1..42a416c 100644 --- a/cmd/subcmd/get.go +++ b/cmd/subcmd/get.go @@ -2,9 +2,11 @@ package subcmd import ( "bytes" + "encoding/hex" "fmt" "os" "path/filepath" + "time" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_irodsfs "github.com/cyverse/go-irodsclient/irods/fs" @@ -38,6 +40,7 @@ func AddGetCommand(rootCmd *cobra.Command) { flag.SetRetryFlags(getCmd) flag.SetDifferentialTransferFlags(getCmd, true) flag.SetChecksumFlags(getCmd, false) + flag.SetTransferReportFlags(getCmd) flag.SetNoRootFlags(getCmd) flag.SetSyncFlags(getCmd) flag.SetDecryptionFlags(getCmd) @@ -78,6 +81,7 @@ func processGetCommand(command *cobra.Command, args []string) error { syncFlagValues := flag.GetSyncFlagValues() decryptionFlagValues := flag.GetDecryptionFlagValues(command) postTransferFlagValues := flag.GetPostTransferFlagValues() + transferReportFlagValues := flag.GetTransferReportFlagValues(command) maxConnectionNum := parallelTransferFlagValues.ThreadNumber + 2 // 2 for metadata op @@ -130,6 +134,11 @@ func processGetCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to get multiple source collections without creating root directory") } + transferReportManager, err := commons.NewTransferReportManager(transferReportFlagValues.Report, transferReportFlagValues.ReportPath, transferReportFlagValues.ReportToStdout) + if err != nil { + return xerrors.Errorf("failed to create transfer report manager: %w", err) + } + parallelJobManager := commons.NewParallelJobManager(filesystem, parallelTransferFlagValues.ThreadNumber, progressFlagValues.ShowProgress, progressFlagValues.ShowFullPath) parallelJobManager.Start() @@ -141,7 +150,7 @@ func processGetCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to make new target path for get %s to %s: %w", sourcePath, targetPath, err) } - err = getOne(parallelJobManager, inputPathMap, sourcePath, newTargetDirPath, forceFlagValues, parallelTransferFlagValues, differentialTransferFlagValues, checksumFlagValues, decryptionFlagValues, postTransferFlagValues) + err = getOne(parallelJobManager, transferReportManager, inputPathMap, sourcePath, newTargetDirPath, forceFlagValues, parallelTransferFlagValues, differentialTransferFlagValues, checksumFlagValues, decryptionFlagValues, postTransferFlagValues) if err != nil { return xerrors.Errorf("failed to perform get %s to %s: %w", sourcePath, targetPath, err) } @@ -179,7 +188,7 @@ func getEncryptionManagerForDecrypt(mode commons.EncryptionMode, decryptionFlagV return manager } -func getOne(parallelJobManager *commons.ParallelJobManager, inputPathMap map[string]bool, sourcePath string, targetPath string, forceFlagValues *flag.ForceFlagValues, parallelTransferFlagValues *flag.ParallelTransferFlagValues, differentialTransferFlagValues *flag.DifferentialTransferFlagValues, checksumFlagValues *flag.ChecksumFlagValues, decryptionFlagValues *flag.DecryptionFlagValues, postTransferFlagValues *flag.PostTransferFlagValues) error { +func getOne(parallelJobManager *commons.ParallelJobManager, transferReportManager *commons.TransferReportManager, inputPathMap map[string]bool, sourcePath string, targetPath string, forceFlagValues *flag.ForceFlagValues, parallelTransferFlagValues *flag.ParallelTransferFlagValues, differentialTransferFlagValues *flag.DifferentialTransferFlagValues, checksumFlagValues *flag.ChecksumFlagValues, decryptionFlagValues *flag.DecryptionFlagValues, postTransferFlagValues *flag.PostTransferFlagValues) error { logger := log.WithFields(log.Fields{ "package": "subcmd", "function": "getOne", @@ -261,25 +270,37 @@ func getOne(parallelJobManager *commons.ParallelJobManager, inputPathMap map[str logger.Debugf("downloading a data object %s to %s", sourcePath, targetFilePath) var downloadErr error + var downloadResult *irodsclient_fs.FileTransferResult // determine how to download + notes := []string{} if parallelTransferFlagValues.SingleTread || parallelTransferFlagValues.ThreadNumber == 1 { - downloadErr = fs.DownloadFileResumable(sourcePath, "", targetFilePath, checksumFlagValues.VerifyChecksum, callbackGet) + downloadResult, downloadErr = fs.DownloadFileResumable(sourcePath, "", targetFilePath, checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "icat") + notes = append(notes, "single-thread") } else if parallelTransferFlagValues.RedirectToResource { - downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) + downloadResult, downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "redirect-to-resource") } else if parallelTransferFlagValues.Icat { - downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) + downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "icat") + notes = append(notes, "multi-thread") } else { // auto if sourceEntry.Size >= commons.RedirectToResourceMinSize { // redirect-to-resource - downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) + downloadResult, downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "redirect-to-resource") } else { - downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) + downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "icat") + notes = append(notes, "multi-thread") } } + transferReportManager.AddTransfer(downloadResult, commons.TransferMethodGet, downloadErr, notes) + if downloadErr != nil { job.Progress(-1, sourceEntry.Size, true) return xerrors.Errorf("failed to download %s to %s: %w", sourcePath, targetFilePath, downloadErr) @@ -324,13 +345,27 @@ func getOne(parallelJobManager *commons.ParallelJobManager, inputPathMap map[str // trx status not exist if differentialTransferFlagValues.NoHash { if targetEntry.Size() == sourceEntry.Size { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + LocalPath: targetFilePath, + LocalSize: targetEntry.Size(), + IrodsPath: sourcePath, + IrodsSize: sourceEntry.Size, + IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"differential", "no_hash", "same file size", "skip"}, + } + + transferReportManager.AddFile(reportFile) + commons.Printf("skip downloading a data object %s. The file already exists!\n", targetFilePath) logger.Debugf("skip downloading a data object %s. The file already exists!", targetFilePath) return nil } - - // delete file to not write to existing file - os.Remove(targetFilePath) } else { if targetEntry.Size() == sourceEntry.Size { if len(sourceEntry.CheckSum) > 0 { @@ -341,29 +376,57 @@ func getOne(parallelJobManager *commons.ParallelJobManager, inputPathMap map[str } if bytes.Equal(sourceEntry.CheckSum, hash) { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + LocalPath: targetFilePath, + LocalSize: targetEntry.Size(), + IrodsPath: sourcePath, + IrodsSize: sourceEntry.Size, + IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"differential", "same hash", "same file size", "skip"}, + } + + transferReportManager.AddFile(reportFile) + commons.Printf("skip downloading a data object %s. The file with the same hash already exists!\n", targetFilePath) logger.Debugf("skip downloading a data object %s. The file with the same hash already exists!", targetFilePath) return nil } } } - - // delete file to not write to existing file - os.Remove(targetFilePath) } } else { if !forceFlagValues.Force { // ask overwrite := commons.InputYN(fmt.Sprintf("file %s already exists. Overwrite?", targetFilePath)) if !overwrite { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + LocalPath: targetFilePath, + LocalSize: targetEntry.Size(), + IrodsPath: sourcePath, + IrodsSize: sourceEntry.Size, + IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"no overwrite", "skip"}, + } + + transferReportManager.AddFile(reportFile) + commons.Printf("skip downloading a data object %s. The file already exists!\n", targetFilePath) logger.Debugf("skip downloading a data object %s. The file already exists!", targetFilePath) return nil } } - - // delete file to not write to existing file - os.Remove(targetFilePath) } } @@ -394,7 +457,7 @@ func getOne(parallelJobManager *commons.ParallelJobManager, inputPathMap map[str commons.MarkPathMap(inputPathMap, targetDirPath) - err = getOne(parallelJobManager, inputPathMap, entry.Path, targetDirPath, forceFlagValues, parallelTransferFlagValues, differentialTransferFlagValues, checksumFlagValues, decryptionFlagValuesCopy, postTransferFlagValues) + err = getOne(parallelJobManager, transferReportManager, inputPathMap, entry.Path, targetDirPath, forceFlagValues, parallelTransferFlagValues, differentialTransferFlagValues, checksumFlagValues, decryptionFlagValuesCopy, postTransferFlagValues) if err != nil { return xerrors.Errorf("failed to perform get %s to %s: %w", entry.Path, targetDirPath, err) } diff --git a/commons/transfer_report.go b/commons/transfer_report.go new file mode 100644 index 0000000..a924ae8 --- /dev/null +++ b/commons/transfer_report.go @@ -0,0 +1,162 @@ +package commons + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "time" + + irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + "golang.org/x/xerrors" +) + +// TransferMethod determines transfer method +type TransferMethod string + +const ( + // TransferMethodGet is for get + TransferMethodGet TransferMethod = "GET" + // TransferMethodPut is for put + TransferMethodPut TransferMethod = "PUT" + // TransferMethodBput is for bput command + TransferMethodBput TransferMethod = "BPUT" + // TransferMethodBputUnknown is for unknown command + TransferMethodBputUnknown TransferMethod = "UNKNOWN" +) + +type TransferReportFile struct { + Method TransferMethod `json:"method"` // get, put, bput ... + + StartAt time.Time `json:"start_time"` + EndAt time.Time `json:"end_at"` + + LocalPath string `json:"local_path"` + IrodsPath string `json:"irods_path"` + ChecksumAlgorithm string `json:"checksum_algorithm"` + LocalSize int64 `json:"local_size"` + LocalChecksum string `json:"local_checksum"` + IrodsSize int64 `json:"irods_size"` + IrodsChecksum string `json:"irods_checksum"` + + Error error `json:"error,omitempty"` + Notes []string `json:"notes"` // additional notes +} + +// GetTransferMethod returns transfer method +func GetTransferMethod(method string) TransferMethod { + switch strings.ToUpper(method) { + case string(TransferMethodGet), "DOWNLOAD", "DOWN": + return TransferMethodGet + case string(TransferMethodPut), "UPLOAD", "UP": + return TransferMethodPut + case string(TransferMethodBput), "BULK_UPLOAD": + return TransferMethodBput + default: + return TransferMethodBputUnknown + } +} + +func NewTransferReportFileFromTransferResult(result *irodsclient_fs.FileTransferResult, method TransferMethod, err error, notes []string) *TransferReportFile { + return &TransferReportFile{ + Method: method, + StartAt: result.StartTime, + EndAt: result.EndTime, + LocalPath: result.LocalPath, + LocalSize: result.LocalSize, + LocalChecksum: hex.EncodeToString(result.LocalCheckSum), + IrodsPath: result.IRODSPath, + IrodsSize: result.IRODSSize, + IrodsChecksum: hex.EncodeToString(result.IRODSCheckSum), + ChecksumAlgorithm: string(result.CheckSumAlgorithm), + Error: err, + Notes: notes, + } +} + +type TransferReportManager struct { + reportPath string + report bool + reportToStdout bool + + writer io.WriteCloser +} + +// NewTransferReportManager creates a new TransferReportManager +func NewTransferReportManager(report bool, reportPath string, reportToStdout bool) (*TransferReportManager, error) { + var writer io.WriteCloser + if !report { + writer = nil + } else if reportToStdout { + // stdout + writer = os.Stdout + } else { + // file + fileWriter, err := os.Create(reportPath) + if err != nil { + return nil, xerrors.Errorf("failed to create a report file %s: %w", reportPath, err) + } + writer = fileWriter + } + + manager := &TransferReportManager{ + report: report, + reportPath: reportPath, + reportToStdout: reportToStdout, + + writer: writer, + } + + return manager, nil +} + +// Release releases resources +func (manager *TransferReportManager) Release() { + if manager.writer != nil { + if !manager.reportToStdout { + manager.writer.Close() + } + + manager.writer = nil + } +} + +// AddFile adds a new file transfer +func (manager *TransferReportManager) AddFile(file *TransferReportFile) error { + if !manager.report { + return nil + } + + if manager.writer == nil { + return nil + } + + lineOutput := "" + if manager.reportToStdout { + // line print + fmt.Printf("[%s]\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", file.Method, file.StartAt, file.EndAt, file.LocalPath, file.IrodsPath, file.LocalSize, file.IrodsSize, file.LocalChecksum, file.IrodsChecksum) + } else { + // json + fileBytes, err := json.Marshal(file) + if err != nil { + return err + } + + lineOutput = string(fileBytes) + "\n" + } + + _, err := manager.writer.Write([]byte(lineOutput)) + if err != nil { + return err + } + + return nil +} + +// AddTransfer adds a new file transfer +func (manager *TransferReportManager) AddTransfer(result *irodsclient_fs.FileTransferResult, method TransferMethod, err error, notes []string) error { + file := NewTransferReportFileFromTransferResult(result, method, err, notes) + return manager.AddFile(file) +} diff --git a/go.mod b/go.mod index 9ada056..06f1a43 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/creativeprojects/go-selfupdate v1.0.1 - github.com/cyverse/go-irodsclient v0.14.11 + github.com/cyverse/go-irodsclient v0.14.12 github.com/dustin/go-humanize v1.0.1 github.com/gliderlabs/ssh v0.3.5 github.com/jedib0t/go-pretty/v6 v6.3.1 diff --git a/go.sum b/go.sum index ea3d1db..87ad869 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creativeprojects/go-selfupdate v1.0.1 h1:5Un4MTv4puCR5GBgkDLC14J72fljGuMC60E63cFGq1o= github.com/creativeprojects/go-selfupdate v1.0.1/go.mod h1:nm7AWUJfrfYt/SB97NAcMhR0KEpPqlrVHXkWFti+ezw= -github.com/cyverse/go-irodsclient v0.14.11 h1:pPAo3UOncFon/TsIjAAUT+rDTfJ6NRux6tr1fmz7Fx0= -github.com/cyverse/go-irodsclient v0.14.11/go.mod h1:eBXha3cwfrM0p1ijYVqsrLJQHpRwTfpA4c5dKCQsQFc= +github.com/cyverse/go-irodsclient v0.14.12 h1:CTrS/pl9ADAuF5De1s4eB2Y8et29SSae7KYW09M8PZc= +github.com/cyverse/go-irodsclient v0.14.12/go.mod h1:eBXha3cwfrM0p1ijYVqsrLJQHpRwTfpA4c5dKCQsQFc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 8fd3408f38f27fa743dc88df5e5a81f1e09da457 Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Thu, 1 Aug 2024 10:38:40 -0700 Subject: [PATCH 02/16] wip: refactoring --- cmd/flag/bundle_transfer.go | 6 +- cmd/subcmd/addmeta.go | 94 ++-- cmd/subcmd/bclean.go | 79 +++- cmd/subcmd/bput.go | 51 +++ cmd/subcmd/bun.go | 97 +++-- cmd/subcmd/get.go | 827 +++++++++++++++++++++--------------- cmd/subcmd/init.go | 49 ++- cmd/subcmd/ls.go | 234 ++++++---- cmd/subcmd/put.go | 14 +- commons/bundle_transfer.go | 46 +- commons/parallel.go | 16 +- commons/transfer_report.go | 6 + go.mod | 2 +- go.sum | 2 + 14 files changed, 958 insertions(+), 565 deletions(-) diff --git a/cmd/flag/bundle_transfer.go b/cmd/flag/bundle_transfer.go index 4340b27..d094334 100644 --- a/cmd/flag/bundle_transfer.go +++ b/cmd/flag/bundle_transfer.go @@ -13,7 +13,7 @@ type BundleTempFlagValues struct { IRODSTempPath string } -type BundleClearFlagVlaues struct { +type BundleClearFlagValues struct { Clear bool } @@ -26,7 +26,7 @@ type BundleConfigFlagValues struct { var ( bundleTempFlagValues BundleTempFlagValues - bundleClearFlagValues BundleClearFlagVlaues + bundleClearFlagValues BundleClearFlagValues bundleConfigFlagValues BundleConfigFlagValues ) @@ -43,7 +43,7 @@ func SetBundleClearFlags(command *cobra.Command) { command.Flags().BoolVar(&bundleClearFlagValues.Clear, "clear", false, "Clear stale bundle files") } -func GetBundleClearFlagValues() *BundleClearFlagVlaues { +func GetBundleClearFlagValues() *BundleClearFlagValues { return &bundleClearFlagValues } diff --git a/cmd/subcmd/addmeta.go b/cmd/subcmd/addmeta.go index f5c7e7c..5b4f6af 100644 --- a/cmd/subcmd/addmeta.go +++ b/cmd/subcmd/addmeta.go @@ -2,6 +2,7 @@ package subcmd import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -28,7 +29,47 @@ func AddAddmetaCommand(rootCmd *cobra.Command) { } func processAddmetaCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + addMeta, err := NewAddMetaCommand(command, args) + if err != nil { + return err + } + + return addMeta.Process() +} + +type AddMetaCommand struct { + command *cobra.Command + + targetObjectFlagValues *flag.TargetObjectFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + attribute string + value string + unit string +} + +func NewAddMetaCommand(command *cobra.Command, args []string) (*AddMetaCommand, error) { + addMeta := &AddMetaCommand{ + command: command, + + targetObjectFlagValues: flag.GetTargetObjectFlagValues(command), + } + + // get avu + addMeta.attribute = args[0] + addMeta.value = args[1] + addMeta.unit = "" + if len(args) >= 3 { + addMeta.unit = args[2] + } + + return addMeta, nil +} + +func (addMeta *AddMetaCommand) Process() error { + cont, err := flag.ProcessCommonFlags(addMeta.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -43,37 +84,27 @@ func processAddmetaCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - targetObjectFlagValues := flag.GetTargetObjectFlagValues(command) - - // Create a connection - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + // Create a file system + addMeta.account = commons.GetAccount() + addMeta.filesystem, err = commons.GetIRODSFSClient(addMeta.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer addMeta.filesystem.Release() - defer filesystem.Release() - - // get avu - attr := args[0] - value := args[1] - unit := "" - if len(args) >= 3 { - unit = args[2] - } - - if targetObjectFlagValues.PathUpdated { - err = addMetaToPath(filesystem, targetObjectFlagValues.Path, attr, value, unit) + // add meta + if addMeta.targetObjectFlagValues.PathUpdated { + err = addMeta.addMetaToPath(addMeta.targetObjectFlagValues.Path, addMeta.attribute, addMeta.value, addMeta.unit) if err != nil { return err } - } else if targetObjectFlagValues.UserUpdated { - err = addMetaToUser(filesystem, targetObjectFlagValues.User, attr, value, unit) + } else if addMeta.targetObjectFlagValues.UserUpdated { + err = addMeta.addMetaToUser(addMeta.targetObjectFlagValues.User, addMeta.attribute, addMeta.value, addMeta.unit) if err != nil { return err } - } else if targetObjectFlagValues.ResourceUpdated { - err = addMetaToResource(filesystem, targetObjectFlagValues.Resource, attr, value, unit) + } else if addMeta.targetObjectFlagValues.ResourceUpdated { + err = addMeta.addMetaToResource(addMeta.targetObjectFlagValues.Resource, addMeta.attribute, addMeta.value, addMeta.unit) if err != nil { return err } @@ -85,20 +116,21 @@ func processAddmetaCommand(command *cobra.Command, args []string) error { return nil } -func addMetaToPath(fs *irodsclient_fs.FileSystem, targetPath string, attribute string, value string, unit string) error { +func (addMeta *AddMetaCommand) addMetaToPath(targetPath string, attribute string, value string, unit string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "AddMetaCommand", "function": "addMetaToPath", }) - logger.Debugf("add metadata to path %s (attr %s, value %s, unit %s)", targetPath, attribute, value, unit) - cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - err := fs.AddMetadata(targetPath, attribute, value, unit) + logger.Debugf("add metadata to path %s (attr %s, value %s, unit %s)", targetPath, attribute, value, unit) + + err := addMeta.filesystem.AddMetadata(targetPath, attribute, value, unit) if err != nil { return xerrors.Errorf("failed to add metadata to path %s (attr %s, value %s, unit %s): %w", targetPath, attribute, value, unit, err) } @@ -106,15 +138,16 @@ func addMetaToPath(fs *irodsclient_fs.FileSystem, targetPath string, attribute s return nil } -func addMetaToUser(fs *irodsclient_fs.FileSystem, username string, attribute string, value string, unit string) error { +func (addMeta *AddMetaCommand) addMetaToUser(username string, attribute string, value string, unit string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "AddMetaCommand", "function": "addMetaToUser", }) logger.Debugf("add metadata to user %s (attr %s, value %s, unit %s)", username, attribute, value, unit) - err := fs.AddUserMetadata(username, attribute, value, unit) + err := addMeta.filesystem.AddUserMetadata(username, attribute, value, unit) if err != nil { return xerrors.Errorf("failed to add metadata to user %s (attr %s, value %s, unit %s): %w", username, attribute, value, unit, err) } @@ -122,15 +155,16 @@ func addMetaToUser(fs *irodsclient_fs.FileSystem, username string, attribute str return nil } -func addMetaToResource(fs *irodsclient_fs.FileSystem, resource string, attribute string, value string, unit string) error { +func (addMeta *AddMetaCommand) addMetaToResource(resource string, attribute string, value string, unit string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "AddMetaCommand", "function": "addMetaToResource", }) logger.Debugf("add metadata to resource %s (attr %s, value %s, unit %s)", resource, attribute, value, unit) - err := fs.AddUserMetadata(resource, attribute, value, unit) + err := addMeta.filesystem.AddUserMetadata(resource, attribute, value, unit) if err != nil { return xerrors.Errorf("failed to add metadata to resource %s (attr %s, value %s, unit %s): %w", resource, attribute, value, unit, err) } diff --git a/cmd/subcmd/bclean.go b/cmd/subcmd/bclean.go index 7ab4bf0..ee94c27 100644 --- a/cmd/subcmd/bclean.go +++ b/cmd/subcmd/bclean.go @@ -2,6 +2,7 @@ package subcmd import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -29,12 +30,48 @@ func AddBcleanCommand(rootCmd *cobra.Command) { } func processBcleanCommand(command *cobra.Command, args []string) error { + bclean, err := NewBcleanCommand(command, args) + if err != nil { + return err + } + + return bclean.Process() +} + +type BcleanCommand struct { + command *cobra.Command + + forceFlagValues *flag.ForceFlagValues + bundleTempFlagValues *flag.BundleTempFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + targetPaths []string +} + +func NewBcleanCommand(command *cobra.Command, args []string) (*BcleanCommand, error) { + bclean := &BcleanCommand{ + command: command, + + forceFlagValues: flag.GetForceFlagValues(), + bundleTempFlagValues: flag.GetBundleTempFlagValues(), + } + + // path + bclean.targetPaths = args[:] + + return bclean, nil +} + +func (bclean *BcleanCommand) Process() error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "processBcleanCommand", + "struct": "BcleanCommand", + "function": "Process", }) - cont, err := flag.ProcessCommonFlags(command) + cont, err := flag.ProcessCommonFlags(bclean.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -49,41 +86,41 @@ func processBcleanCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - forceFlagValues := flag.GetForceFlagValues() - bundleTempFlagValues := flag.GetBundleTempFlagValues() - - // clear local - commons.CleanUpOldLocalBundles(bundleTempFlagValues.LocalTempPath, forceFlagValues.Force) - // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + bclean.account = commons.GetAccount() + bclean.filesystem, err = commons.GetIRODSFSClient(bclean.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer bclean.filesystem.Release() + + // run + // clear local + commons.CleanUpOldLocalBundles(bclean.bundleTempFlagValues.LocalTempPath, bclean.forceFlagValues.Force) - defer filesystem.Release() + // clear remote + if len(bclean.bundleTempFlagValues.IRODSTempPath) > 0 { + logger.Debugf("clearing irods temp dir %s", bclean.bundleTempFlagValues.IRODSTempPath) - if len(bundleTempFlagValues.IRODSTempPath) > 0 { - logger.Debugf("clearing irods temp dir %s", bundleTempFlagValues.IRODSTempPath) - commons.CleanUpOldIRODSBundles(filesystem, bundleTempFlagValues.IRODSTempPath, true, forceFlagValues.Force) + commons.CleanUpOldIRODSBundles(bclean.filesystem, bclean.bundleTempFlagValues.IRODSTempPath, true, bclean.forceFlagValues.Force) } else { userHome := commons.GetHomeDir() homeStagingDir := commons.GetDefaultStagingDir(userHome) - commons.CleanUpOldIRODSBundles(filesystem, homeStagingDir, true, forceFlagValues.Force) + commons.CleanUpOldIRODSBundles(bclean.filesystem, homeStagingDir, true, bclean.forceFlagValues.Force) } - for _, targetPath := range args { - bcleanOne(filesystem, targetPath, forceFlagValues) + for _, targetPath := range bclean.targetPaths { + bclean.cleanOne(targetPath) } return nil } -func bcleanOne(fs *irodsclient_fs.FileSystem, targetPath string, forceFlagValues *flag.ForceFlagValues) { +func (bclean *BcleanCommand) cleanOne(targetPath string) { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "bcleanOne", + "struct": "BcleanCommand", + "function": "cleanOne", }) cwd := commons.GetCWD() @@ -94,12 +131,12 @@ func bcleanOne(fs *irodsclient_fs.FileSystem, targetPath string, forceFlagValues if commons.IsStagingDirInTargetPath(targetPath) { // target is staging dir logger.Debugf("clearing irods target dir %s", targetPath) - commons.CleanUpOldIRODSBundles(fs, targetPath, true, forceFlagValues.Force) + commons.CleanUpOldIRODSBundles(bclean.filesystem, targetPath, true, bclean.forceFlagValues.Force) return } stagingDirPath := commons.GetDefaultStagingDirInTargetPath(targetPath) logger.Debugf("clearing irods target dir %s", stagingDirPath) - commons.CleanUpOldIRODSBundles(fs, stagingDirPath, true, forceFlagValues.Force) + commons.CleanUpOldIRODSBundles(bclean.filesystem, stagingDirPath, true, bclean.forceFlagValues.Force) } diff --git a/cmd/subcmd/bput.go b/cmd/subcmd/bput.go index 92a1694..d375ad0 100644 --- a/cmd/subcmd/bput.go +++ b/cmd/subcmd/bput.go @@ -40,6 +40,57 @@ func AddBputCommand(rootCmd *cobra.Command) { rootCmd.AddCommand(bputCmd) } +type BputCommand struct { + command *cobra.Command + + bundleTempFlagValues *flag.BundleTempFlagValues + bundleClearFlagValues *flag.BundleClearFlagValues + bundleConfigFlagValues *flag.BundleConfigFlagValues + parallelTransferFlagValues *flag.ParallelTransferFlagValues + progressFlagValues *flag.ProgressFlagValues + retryFlagValues *flag.RetryFlagValues + differentialTransferFlagValues *flag.DifferentialTransferFlagValues + noRootFlagValues *flag.NoRootFlagValues + syncFlagValues *flag.SyncFlagValues + + maxConnectionNum int + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + sourcePaths []string + targetPath string +} + +func NewBputCommand(command *cobra.Command, args []string) (*BputCommand, error) { + bput := &BputCommand{ + command: command, + + bundleTempFlagValues: flag.GetBundleTempFlagValues(), + bundleClearFlagValues: flag.GetBundleClearFlagValues(), + bundleConfigFlagValues: flag.GetBundleConfigFlagValues(), + parallelTransferFlagValues: flag.GetParallelTransferFlagValues(), + progressFlagValues: flag.GetProgressFlagValues(), + retryFlagValues: flag.GetRetryFlagValues(), + differentialTransferFlagValues: flag.GetDifferentialTransferFlagValues(), + noRootFlagValues: flag.GetNoRootFlagValues(), + syncFlagValues: flag.GetSyncFlagValues(), + } + + bput.maxConnectionNum = bput.parallelTransferFlagValues.ThreadNumber + 2 + 2 // 2 for metadata op, 2 for extraction + + // path + bput.targetPath = "./" + bput.sourcePaths = args[:] + + if len(args) >= 2 { + bput.targetPath = args[len(args)-1] + bput.sourcePaths = args[:len(args)-1] + } + + return bput, nil +} + func processBputCommand(command *cobra.Command, args []string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", diff --git a/cmd/subcmd/bun.go b/cmd/subcmd/bun.go index 79c73af..080674a 100644 --- a/cmd/subcmd/bun.go +++ b/cmd/subcmd/bun.go @@ -32,7 +32,48 @@ func AddBunCommand(rootCmd *cobra.Command) { } func processBunCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + bun, err := NewBunCommand(command, args) + if err != nil { + return err + } + + return bun.Process() +} + +type BunCommand struct { + command *cobra.Command + + forceFlagValues *flag.ForceFlagValues + bundleFlagValues *flag.BundleFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + sourcePaths []string + targetPath string +} + +func NewBunCommand(command *cobra.Command, args []string) (*BunCommand, error) { + bun := &BunCommand{ + command: command, + + forceFlagValues: flag.GetForceFlagValues(), + bundleFlagValues: flag.GetBundleFlagValues(), + } + + // path + bun.targetPath = args[len(args)-1] + bun.sourcePaths = args[:len(args)-1] + + if !bun.bundleFlagValues.Extract { + return nil, xerrors.Errorf("support only extract mode") + } + + return bun, nil +} + +func (bun *BunCommand) Process() error { + cont, err := flag.ProcessCommonFlags(bun.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -47,28 +88,20 @@ func processBunCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - forceFlagValues := flag.GetForceFlagValues() - bundleFlagValues := flag.GetBundleFlagValues() - - if !bundleFlagValues.Extract { - return xerrors.Errorf("support only extract mode") - } - // Create a file system account := commons.GetAccount() filesystem, err := commons.GetIRODSFSClient(account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } - defer filesystem.Release() - targetPath := args[len(args)-1] - for _, sourcePath := range args[:len(args)-1] { - if bundleFlagValues.Extract { - err = extractOne(filesystem, sourcePath, targetPath, bundleFlagValues, forceFlagValues) + // run + for _, sourcePath := range bun.sourcePaths { + if bun.bundleFlagValues.Extract { + err = bun.extractOne(sourcePath, bun.targetPath) if err != nil { - return xerrors.Errorf("failed to perform bun %s to %s: %w", sourcePath, targetPath, err) + return xerrors.Errorf("failed to perform bun %s to %s: %w", sourcePath, bun.targetPath, err) } } } @@ -76,7 +109,7 @@ func processBunCommand(command *cobra.Command, args []string) error { return nil } -func getDataType(irodsPath string, dataType string) (irodsclient_types.DataType, error) { +func (bun *BunCommand) getDataType(irodsPath string, dataType string) (irodsclient_types.DataType, error) { switch strings.ToLower(dataType) { case "tar", "t", "tar file": return irodsclient_types.TAR_FILE_DT, nil @@ -108,9 +141,10 @@ func getDataType(irodsPath string, dataType string) (irodsclient_types.DataType, } } -func extractOne(filesystem *irodsclient_fs.FileSystem, sourcePath string, targetPath string, bundleFlagValues *flag.BundleFlagValues, forceFlagValues *flag.ForceFlagValues) error { +func (bun *BunCommand) extractOne(sourcePath string, targetPath string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "BunCommand", "function": "extractOne", }) @@ -120,12 +154,12 @@ func extractOne(filesystem *irodsclient_fs.FileSystem, sourcePath string, target sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - sourceEntry, err := filesystem.Stat(sourcePath) + sourceEntry, err := bun.filesystem.Stat(sourcePath) if err != nil { return xerrors.Errorf("failed to stat %s: %w", sourcePath, err) } - targetEntry, err := filesystem.Stat(targetPath) + targetEntry, err := bun.filesystem.Stat(targetPath) if err != nil { if !irodsclient_types.IsFileNotFoundError(err) { return xerrors.Errorf("failed to stat %s: %w", targetPath, err) @@ -136,21 +170,22 @@ func extractOne(filesystem *irodsclient_fs.FileSystem, sourcePath string, target } } - if sourceEntry.Type == irodsclient_fs.FileEntry { - // file - logger.Debugf("extracting a data object %s to %s", sourcePath, targetPath) + if sourceEntry.Type != irodsclient_fs.FileEntry { + return xerrors.Errorf("source %s must be a data object", sourcePath) + } - dt, err := getDataType(sourcePath, bundleFlagValues.DataType) - if err != nil { - return xerrors.Errorf("failed to get type %s: %w", sourcePath, err) - } + // file + logger.Debugf("extracting a data object %s to %s", sourcePath, targetPath) - err = filesystem.ExtractStructFile(sourcePath, targetPath, "", dt, forceFlagValues.Force, bundleFlagValues.BulkRegistration) - if err != nil { - return xerrors.Errorf("failed to extract file %s to %s: %w", sourcePath, targetPath, err) - } - } else { - return xerrors.Errorf("source %s must be a data object", sourcePath) + dt, err := bun.getDataType(sourcePath, bun.bundleFlagValues.DataType) + if err != nil { + return xerrors.Errorf("failed to get type %s: %w", sourcePath, err) } + + err = bun.filesystem.ExtractStructFile(sourcePath, targetPath, "", dt, bun.forceFlagValues.Force, bun.bundleFlagValues.BulkRegistration) + if err != nil { + return xerrors.Errorf("failed to extract file %s to %s: %w", sourcePath, targetPath, err) + } + return nil } diff --git a/cmd/subcmd/get.go b/cmd/subcmd/get.go index 42a416c..f669d1c 100644 --- a/cmd/subcmd/get.go +++ b/cmd/subcmd/get.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "fmt" "os" - "path/filepath" "time" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" @@ -49,13 +48,81 @@ func AddGetCommand(rootCmd *cobra.Command) { rootCmd.AddCommand(getCmd) } -func processGetCommand(command *cobra.Command, args []string) error { +type GetCommand struct { + command *cobra.Command + + forceFlagValues *flag.ForceFlagValues + ticketAccessFlagValues *flag.TicketAccessFlagValues + parallelTransferFlagValues *flag.ParallelTransferFlagValues + progressFlagValues *flag.ProgressFlagValues + retryFlagValues *flag.RetryFlagValues + differentialTransferFlagValues *flag.DifferentialTransferFlagValues + checksumFlagValues *flag.ChecksumFlagValues + noRootFlagValues *flag.NoRootFlagValues + syncFlagValues *flag.SyncFlagValues + decryptionFlagValues *flag.DecryptionFlagValues + postTransferFlagValues *flag.PostTransferFlagValues + transferReportFlagValues *flag.TransferReportFlagValues + + maxConnectionNum int + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + sourcePaths []string + targetPath string + + parallelJobManager *commons.ParallelJobManager + transferReportManager *commons.TransferReportManager + inputPathMap map[string]bool +} + +func NewGetCommand(command *cobra.Command, args []string) (*GetCommand, error) { + get := &GetCommand{ + command: command, + + forceFlagValues: flag.GetForceFlagValues(), + ticketAccessFlagValues: flag.GetTicketAccessFlagValues(), + parallelTransferFlagValues: flag.GetParallelTransferFlagValues(), + progressFlagValues: flag.GetProgressFlagValues(), + retryFlagValues: flag.GetRetryFlagValues(), + differentialTransferFlagValues: flag.GetDifferentialTransferFlagValues(), + checksumFlagValues: flag.GetChecksumFlagValues(), + noRootFlagValues: flag.GetNoRootFlagValues(), + syncFlagValues: flag.GetSyncFlagValues(), + decryptionFlagValues: flag.GetDecryptionFlagValues(command), + postTransferFlagValues: flag.GetPostTransferFlagValues(), + transferReportFlagValues: flag.GetTransferReportFlagValues(command), + + inputPathMap: map[string]bool{}, + } + + get.maxConnectionNum = get.parallelTransferFlagValues.ThreadNumber + 2 // 2 for metadata op + + // path + get.targetPath = "./" + get.sourcePaths = args[:] + + if len(args) >= 2 { + get.targetPath = args[len(args)-1] + get.sourcePaths = args[:len(args)-1] + } + + if get.noRootFlagValues.NoRoot && len(get.sourcePaths) > 1 { + return nil, xerrors.Errorf("failed to get multiple source collections without creating root directory") + } + + return get, nil +} + +func (get *GetCommand) Process() error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "processGetCommand", + "struct": "GetCommand", + "function": "Process", }) - cont, err := flag.ProcessCommonFlags(command) + cont, err := flag.ProcessCommonFlags(get.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -70,34 +137,12 @@ func processGetCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - forceFlagValues := flag.GetForceFlagValues() - ticketAccessFlagValues := flag.GetTicketAccessFlagValues() - parallelTransferFlagValues := flag.GetParallelTransferFlagValues() - progressFlagValues := flag.GetProgressFlagValues() - retryFlagValues := flag.GetRetryFlagValues() - differentialTransferFlagValues := flag.GetDifferentialTransferFlagValues() - checksumFlagValues := flag.GetChecksumFlagValues() - noRootFlagValues := flag.GetNoRootFlagValues() - syncFlagValues := flag.GetSyncFlagValues() - decryptionFlagValues := flag.GetDecryptionFlagValues(command) - postTransferFlagValues := flag.GetPostTransferFlagValues() - transferReportFlagValues := flag.GetTransferReportFlagValues(command) - - maxConnectionNum := parallelTransferFlagValues.ThreadNumber + 2 // 2 for metadata op - - if retryFlagValues.RetryNumber > 0 && !retryFlagValues.RetryChild { - err = commons.RunWithRetry(retryFlagValues.RetryNumber, retryFlagValues.RetryIntervalSeconds) - if err != nil { - return xerrors.Errorf("failed to run with retry %d: %w", retryFlagValues.RetryNumber, err) - } - return nil - } - + // config appConfig := commons.GetConfig() syncAccount := false - if len(ticketAccessFlagValues.Name) > 0 { - logger.Debugf("use ticket: %s", ticketAccessFlagValues.Name) - appConfig.Ticket = ticketAccessFlagValues.Name + if len(get.ticketAccessFlagValues.Name) > 0 { + logger.Debugf("use ticket: %s", get.ticketAccessFlagValues.Name) + appConfig.Ticket = get.ticketAccessFlagValues.Name syncAccount = true } @@ -108,426 +153,491 @@ func processGetCommand(command *cobra.Command, args []string) error { } } + // handle retry + if get.retryFlagValues.RetryNumber > 0 && !get.retryFlagValues.RetryChild { + err := commons.RunWithRetry(get.retryFlagValues.RetryNumber, get.retryFlagValues.RetryIntervalSeconds) + if err != nil { + return xerrors.Errorf("failed to run with retry %d: %w", get.retryFlagValues.RetryNumber, err) + } + + return nil + } + // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClientAdvanced(account, maxConnectionNum, parallelTransferFlagValues.TCPBufferSize) + get.account = commons.GetAccount() + get.filesystem, err = commons.GetIRODSFSClientAdvanced(get.account, get.maxConnectionNum, get.parallelTransferFlagValues.TCPBufferSize) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } - defer filesystem.Release() + defer get.filesystem.Release() - // set default key for decryption - if len(decryptionFlagValues.Key) == 0 { - decryptionFlagValues.Key = account.Password + // transfer report + get.transferReportManager, err = commons.NewTransferReportManager(get.transferReportFlagValues.Report, get.transferReportFlagValues.ReportPath, get.transferReportFlagValues.ReportToStdout) + if err != nil { + return xerrors.Errorf("failed to create transfer report manager: %w", err) } - targetPath := "./" - sourcePaths := args[:] + defer get.transferReportManager.Release() - if len(args) >= 2 { - targetPath = args[len(args)-1] - sourcePaths = args[:len(args)-1] + // set default key for decryption + if len(get.decryptionFlagValues.Key) == 0 { + get.decryptionFlagValues.Key = get.account.Password } - if noRootFlagValues.NoRoot && len(sourcePaths) > 1 { + if get.noRootFlagValues.NoRoot && len(get.sourcePaths) > 1 { return xerrors.Errorf("failed to get multiple source collections without creating root directory") } - transferReportManager, err := commons.NewTransferReportManager(transferReportFlagValues.Report, transferReportFlagValues.ReportPath, transferReportFlagValues.ReportToStdout) + // parallel job manager + get.parallelJobManager = commons.NewParallelJobManager(get.filesystem, get.parallelTransferFlagValues.ThreadNumber, get.progressFlagValues.ShowProgress, get.progressFlagValues.ShowFullPath) + get.parallelJobManager.Start() + + // run + for _, sourcePath := range get.sourcePaths { + newSourcePath, newTargetDirPath, err := get.makeSourceAndTargetDirPath(sourcePath, get.targetPath) + if err != nil { + return xerrors.Errorf("failed to make new target path for get %s to %s: %w", sourcePath, get.targetPath, err) + } + + err = get.getOne(newSourcePath, newTargetDirPath, get.decryptionFlagValues.Decryption) + if err != nil { + return xerrors.Errorf("failed to perform get %s to %s: %w", newSourcePath, newTargetDirPath, err) + } + } + + get.parallelJobManager.DoneScheduling() + err = get.parallelJobManager.Wait() if err != nil { - return xerrors.Errorf("failed to create transfer report manager: %w", err) + return xerrors.Errorf("failed to perform parallel jobs: %w", err) } - parallelJobManager := commons.NewParallelJobManager(filesystem, parallelTransferFlagValues.ThreadNumber, progressFlagValues.ShowProgress, progressFlagValues.ShowFullPath) - parallelJobManager.Start() + return nil +} - inputPathMap := map[string]bool{} +func (get *GetCommand) makeSourceAndTargetDirPath(sourcePath string, targetPath string) (string, string, error) { + cwd := commons.GetCWD() + home := commons.GetHomeDir() + zone := commons.GetZone() + sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) + targetPath = commons.MakeLocalPath(targetPath) - for _, sourcePath := range sourcePaths { - newTargetDirPath, err := makeGetTargetDirPath(filesystem, sourcePath, targetPath, noRootFlagValues.NoRoot) - if err != nil { - return xerrors.Errorf("failed to make new target path for get %s to %s: %w", sourcePath, targetPath, err) - } + sourceEntry, err := get.filesystem.Stat(sourcePath) + if err != nil { + return "", "", xerrors.Errorf("failed to stat %s: %w", sourcePath, err) + } - err = getOne(parallelJobManager, transferReportManager, inputPathMap, sourcePath, newTargetDirPath, forceFlagValues, parallelTransferFlagValues, differentialTransferFlagValues, checksumFlagValues, decryptionFlagValues, postTransferFlagValues) + if sourceEntry.Type == irodsclient_fs.FileEntry { + // file + targetFilePath := commons.MakeTargetLocalFilePath(sourcePath, targetPath) + targetDirPath := commons.GetDir(targetFilePath) + _, err := os.Stat(targetDirPath) if err != nil { - return xerrors.Errorf("failed to perform get %s to %s: %w", sourcePath, targetPath, err) + if os.IsNotExist(err) { + return "", "", irodsclient_types.NewFileNotFoundError(targetDirPath) + } + + return "", "", xerrors.Errorf("failed to stat dir %s: %w", targetDirPath, err) } + + return sourcePath, targetDirPath, nil } - parallelJobManager.DoneScheduling() - err = parallelJobManager.Wait() + // dir + _, err = os.Stat(targetPath) if err != nil { - return xerrors.Errorf("failed to perform parallel jobs: %w", err) + if os.IsNotExist(err) { + return "", "", irodsclient_types.NewFileNotFoundError(targetPath) + } + + return "", "", xerrors.Errorf("failed to stat dir %s: %w", targetPath, err) } - // delete extra - if syncFlagValues.Delete { - logger.Infof("deleting extra files and dirs under %s", targetPath) + targetDirPath := targetPath - err = getDeleteExtra(inputPathMap, targetPath) + if !get.noRootFlagValues.NoRoot { + // make target dir + targetDirPath = commons.MakeTargetLocalFilePath(sourceEntry.Path, targetDirPath) + err = os.MkdirAll(targetDirPath, 0766) if err != nil { - return xerrors.Errorf("failed to delete extra files: %w", err) + return "", "", xerrors.Errorf("failed to make dir %s: %w", targetDirPath, err) } } - return nil + return sourcePath, targetDirPath, nil } -func getEncryptionManagerForDecrypt(mode commons.EncryptionMode, decryptionFlagValues *flag.DecryptionFlagValues) *commons.EncryptionManager { +func (get *GetCommand) getEncryptionManagerForDecryption(mode commons.EncryptionMode) *commons.EncryptionManager { manager := commons.NewEncryptionManager(mode) switch mode { case commons.EncryptionModeWinSCP, commons.EncryptionModePGP: - manager.SetKey([]byte(decryptionFlagValues.Key)) + manager.SetKey([]byte(get.decryptionFlagValues.Key)) case commons.EncryptionModeSSH: - manager.SetPublicPrivateKey(decryptionFlagValues.PrivateKeyPath) + manager.SetPublicPrivateKey(get.decryptionFlagValues.PrivateKeyPath) } return manager } -func getOne(parallelJobManager *commons.ParallelJobManager, transferReportManager *commons.TransferReportManager, inputPathMap map[string]bool, sourcePath string, targetPath string, forceFlagValues *flag.ForceFlagValues, parallelTransferFlagValues *flag.ParallelTransferFlagValues, differentialTransferFlagValues *flag.DifferentialTransferFlagValues, checksumFlagValues *flag.ChecksumFlagValues, decryptionFlagValues *flag.DecryptionFlagValues, postTransferFlagValues *flag.PostTransferFlagValues) error { +func (get *GetCommand) getOne(sourcePath string, targetPath string, requireDecryption bool) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "GetCommand", "function": "getOne", }) - cwd := commons.GetCWD() - home := commons.GetHomeDir() - zone := commons.GetZone() - sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) - targetPath = commons.MakeLocalPath(targetPath) - - filesystem := parallelJobManager.GetFilesystem() - - sourceEntry, err := filesystem.Stat(sourcePath) + sourceEntry, err := get.filesystem.Stat(sourcePath) if err != nil { return xerrors.Errorf("failed to stat %s: %w", sourcePath, err) } - // load encryption config from meta - if !decryptionFlagValues.NoDecryption && !decryptionFlagValues.IgnoreMeta { - sourceDir := sourcePath - if !sourceEntry.IsDir() { - sourceDir = commons.GetDir(sourcePath) - } - - encryptionConfig := commons.GetEncryptionConfigFromMeta(filesystem, sourceDir) - - if encryptionConfig.Required { - decryptionFlagValues.Decryption = encryptionConfig.Required - } + // load encryption config + if !requireDecryption { + requireDecryption = get.requireDecryptionByMeta(sourceEntry) } - originalSourcePath := sourcePath + // if source is dir, recursive + if sourceEntry.Type == irodsclient_fs.DirectoryEntry { + // dir + logger.Debugf("downloading a collection %s to %s", sourcePath, targetPath) - if sourceEntry.Type == irodsclient_fs.FileEntry { - // file - targetFilePath := commons.MakeTargetLocalFilePath(sourcePath, targetPath) - decryptedTargetFilePath := targetFilePath + entries, err := filesystem.List(sourceEntry.Path) + if err != nil { + return xerrors.Errorf("failed to list dir %s: %w", sourceEntry.Path, err) + } - // decrypt first if necessary - encryptionMode := commons.DetectEncryptionMode(sourceEntry.Name) - encryptManager := getEncryptionManagerForDecrypt(encryptionMode, decryptionFlagValues) + for _, entry := range entries { + targetDirPath := targetPath + if entry.Type != irodsclient_fs.FileEntry { + // dir + targetDirPath = commons.MakeTargetLocalFilePath(entry.Path, targetPath) + err = os.MkdirAll(targetDirPath, 0766) + if err != nil { + return xerrors.Errorf("failed to make dir %s: %w", targetDirPath, err) + } + } - if decryptionFlagValues.Decryption && encryptionMode != commons.EncryptionModeUnknown { - targetFilePath = filepath.Join(decryptionFlagValues.TempPath, sourceEntry.Name) + commons.MarkPathMap(get.inputPathMap, targetDirPath) - newFilename, err := encryptManager.DecryptFilename(sourceEntry.Name) + err = get.getOne(entry.Path, targetDirPath, requireDecryption) if err != nil { - return xerrors.Errorf("failed to decrypt %s: %w", targetFilePath, err) + return xerrors.Errorf("failed to perform get %s to %s: %w", entry.Path, targetDirPath, err) } + } - decryptedTargetFilePath = commons.MakeTargetLocalFilePath(newFilename, targetPath) + return nil + } - logger.Debugf("downloading a decrypted file to %s", decryptedTargetFilePath) - } + if sourceEntry.Type != irodsclient_fs.FileEntry { + return xerrors.Errorf("unhandled file entry type %s", sourceEntry.Type) + } - commons.MarkPathMap(inputPathMap, decryptedTargetFilePath) + // file - fileExist := false - targetEntry, err := os.Stat(targetFilePath) - if err != nil { - if !os.IsNotExist(err) { - return xerrors.Errorf("failed to stat %s: %w", targetFilePath, err) - } - } else { - fileExist = true + commons.MarkPathMap(get.inputPathMap, decryptedTargetFilePath) + + fileExist := false + targetEntry, err := os.Stat(targetFilePath) + if err != nil { + if !os.IsNotExist(err) { + return xerrors.Errorf("failed to stat %s: %w", targetFilePath, err) } + } else { + fileExist = true + } - getTask := func(job *commons.ParallelJob) error { - manager := job.GetManager() - fs := manager.GetFilesystem() + getTask := get.getTask(sourcePath, targetFilePath, sourceEntry.Size) - callbackGet := func(processed int64, total int64) { - job.Progress(processed, total, false) - } + if fileExist { + // check transfer status file + trxStatusFilePath := irodsclient_irodsfs.GetDataObjectTransferStatusFilePath(targetFilePath) + trxStatusFileExist := false + _, err = os.Stat(trxStatusFilePath) + if err == nil { + trxStatusFileExist = true + } - job.Progress(0, sourceEntry.Size, false) + if trxStatusFileExist { + // incomplete file - resume downloading + commons.Printf("resume downloading a data object %s\n", targetFilePath) + logger.Debugf("resume downloading a data object %s", targetFilePath) + } else if get.differentialTransferFlagValues.DifferentialTransfer { + // trx status not exist + if get.differentialTransferFlagValues.NoHash { + if targetEntry.Size() == sourceEntry.Size { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + LocalPath: targetFilePath, + LocalSize: targetEntry.Size(), + IrodsPath: sourcePath, + IrodsSize: sourceEntry.Size, + IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"differential", "no_hash", "same file size", "skip"}, + } - logger.Debugf("downloading a data object %s to %s", sourcePath, targetFilePath) + get.transferReportManager.AddFile(reportFile) - var downloadErr error - var downloadResult *irodsclient_fs.FileTransferResult + commons.Printf("skip downloading a data object %s. The file already exists!\n", targetFilePath) + logger.Debugf("skip downloading a data object %s. The file already exists!", targetFilePath) + return nil + } + } else { + if targetEntry.Size() == sourceEntry.Size { + if len(sourceEntry.CheckSum) > 0 { + // compare hash + hash, err := irodsclient_util.HashLocalFile(targetFilePath, string(sourceEntry.CheckSumAlgorithm)) + if err != nil { + return xerrors.Errorf("failed to get hash of %s: %w", targetFilePath, err) + } - // determine how to download - notes := []string{} + if bytes.Equal(sourceEntry.CheckSum, hash) { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + LocalPath: targetFilePath, + LocalSize: targetEntry.Size(), + IrodsPath: sourcePath, + IrodsSize: sourceEntry.Size, + IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"differential", "same hash", "same file size", "skip"}, + } - if parallelTransferFlagValues.SingleTread || parallelTransferFlagValues.ThreadNumber == 1 { - downloadResult, downloadErr = fs.DownloadFileResumable(sourcePath, "", targetFilePath, checksumFlagValues.VerifyChecksum, callbackGet) - notes = append(notes, "icat") - notes = append(notes, "single-thread") - } else if parallelTransferFlagValues.RedirectToResource { - downloadResult, downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) - notes = append(notes, "redirect-to-resource") - } else if parallelTransferFlagValues.Icat { - downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) - notes = append(notes, "icat") - notes = append(notes, "multi-thread") - } else { - // auto - if sourceEntry.Size >= commons.RedirectToResourceMinSize { - // redirect-to-resource - downloadResult, downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) - notes = append(notes, "redirect-to-resource") - } else { - downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetFilePath, 0, checksumFlagValues.VerifyChecksum, callbackGet) - notes = append(notes, "icat") - notes = append(notes, "multi-thread") + get.transferReportManager.AddFile(reportFile) + + commons.Printf("skip downloading a data object %s. The file with the same hash already exists!\n", targetFilePath) + logger.Debugf("skip downloading a data object %s. The file with the same hash already exists!", targetFilePath) + return nil + } + } } } + } else { + if !get.forceFlagValues.Force { + // ask + overwrite := commons.InputYN(fmt.Sprintf("file %s already exists. Overwrite?", targetFilePath)) + if !overwrite { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + LocalPath: targetFilePath, + LocalSize: targetEntry.Size(), + IrodsPath: sourcePath, + IrodsSize: sourceEntry.Size, + IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"no overwrite", "skip"}, + } - transferReportManager.AddTransfer(downloadResult, commons.TransferMethodGet, downloadErr, notes) + get.transferReportManager.AddFile(reportFile) - if downloadErr != nil { - job.Progress(-1, sourceEntry.Size, true) - return xerrors.Errorf("failed to download %s to %s: %w", sourcePath, targetFilePath, downloadErr) + commons.Printf("skip downloading a data object %s. The file already exists!\n", targetFilePath) + logger.Debugf("skip downloading a data object %s. The file already exists!", targetFilePath) + return nil + } } + } + } - logger.Debugf("downloaded a data object %s to %s", sourcePath, targetFilePath) - job.Progress(sourceEntry.Size, sourceEntry.Size, false) + threadsRequired := irodsclient_util.GetNumTasksForParallelTransfer(sourceEntry.Size) + err = get.parallelJobManager.Schedule(sourcePath, getTask, threadsRequired, progress.UnitsBytes) + if err != nil { + return xerrors.Errorf("failed to schedule %s: %w", sourcePath, err) + } - if decryptionFlagValues.Decryption && encryptionMode != commons.EncryptionModeUnknown { - logger.Debugf("decrypt a data object %s to %s", targetFilePath, decryptedTargetFilePath) - err = encryptManager.DecryptFile(targetFilePath, decryptedTargetFilePath) - if err != nil { - return xerrors.Errorf("failed to decrypt %s: %w", targetFilePath, err) - } + logger.Debugf("scheduled a data object download %s to %s", sourcePath, targetPath) + return nil +} - logger.Debugf("removing a temp file %s", targetFilePath) - os.Remove(targetFilePath) - } +func (get *GetCommand) requireDecryptionByMeta(sourceEntry *irodsclient_fs.Entry) bool { + // load encryption config from meta + if !get.decryptionFlagValues.NoDecryption && !get.decryptionFlagValues.IgnoreMeta { + sourceDir := sourceEntry.Path + if !sourceEntry.IsDir() { + sourceDir = commons.GetDir(sourceEntry.Path) + } - if postTransferFlagValues.DeleteOnSuccess { - logger.Debugf("removing source file %s", originalSourcePath) - filesystem.RemoveFile(originalSourcePath, true) - } + encryptionConfig := commons.GetEncryptionConfigFromMeta(get.filesystem, sourceDir) - return nil + if encryptionConfig.Required { + return encryptionConfig.Required } + } - if fileExist { - // check transfer status file - trxStatusFilePath := irodsclient_irodsfs.GetDataObjectTransferStatusFilePath(targetFilePath) - trxStatusFileExist := false - _, err = os.Stat(trxStatusFilePath) - if err == nil { - trxStatusFileExist = true - } + return false +} - if trxStatusFileExist { - // incomplete file - resume downloading - commons.Printf("resume downloading a data object %s\n", targetFilePath) - logger.Debugf("resume downloading a data object %s", targetFilePath) - } else if differentialTransferFlagValues.DifferentialTransfer { - // trx status not exist - if differentialTransferFlagValues.NoHash { - if targetEntry.Size() == sourceEntry.Size { - // skip - now := time.Now() - reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodGet, - StartAt: now, - EndAt: now, - LocalPath: targetFilePath, - LocalSize: targetEntry.Size(), - IrodsPath: sourcePath, - IrodsSize: sourceEntry.Size, - IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"differential", "no_hash", "same file size", "skip"}, - } +func (get *GetCommand) requireDecryption(sourcePath string) bool { + encryptionMode := commons.DetectEncryptionMode(sourcePath) + if get.decryptionFlagValues.Decryption && encryptionMode != commons.EncryptionModeUnknown { + return true + } - transferReportManager.AddFile(reportFile) + return false +} - commons.Printf("skip downloading a data object %s. The file already exists!\n", targetFilePath) - logger.Debugf("skip downloading a data object %s. The file already exists!", targetFilePath) - return nil - } - } else { - if targetEntry.Size() == sourceEntry.Size { - if len(sourceEntry.CheckSum) > 0 { - // compare hash - hash, err := irodsclient_util.HashLocalFile(targetFilePath, string(sourceEntry.CheckSumAlgorithm)) - if err != nil { - return xerrors.Errorf("failed to get hash of %s: %w", targetFilePath, err) - } +func (get *GetCommand) getPathsForDecryption(sourcePath string, targetPath string) (string, string, error) { + sourceFilename := commons.GetBasename(sourcePath) - if bytes.Equal(sourceEntry.CheckSum, hash) { - // skip - now := time.Now() - reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodGet, - StartAt: now, - EndAt: now, - LocalPath: targetFilePath, - LocalSize: targetEntry.Size(), - IrodsPath: sourcePath, - IrodsSize: sourceEntry.Size, - IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"differential", "same hash", "same file size", "skip"}, - } - - transferReportManager.AddFile(reportFile) - - commons.Printf("skip downloading a data object %s. The file with the same hash already exists!\n", targetFilePath) - logger.Debugf("skip downloading a data object %s. The file with the same hash already exists!", targetFilePath) - return nil - } - } - } - } - } else { - if !forceFlagValues.Force { - // ask - overwrite := commons.InputYN(fmt.Sprintf("file %s already exists. Overwrite?", targetFilePath)) - if !overwrite { - // skip - now := time.Now() - reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodGet, - StartAt: now, - EndAt: now, - LocalPath: targetFilePath, - LocalSize: targetEntry.Size(), - IrodsPath: sourcePath, - IrodsSize: sourceEntry.Size, - IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"no overwrite", "skip"}, - } + encryptionMode := commons.DetectEncryptionMode(sourceFilename) + encryptManager := get.getEncryptionManagerForDecryption(encryptionMode) - transferReportManager.AddFile(reportFile) + if get.requireDecryption(sourcePath) { + tempFilePath := commons.MakeTargetLocalFilePath(sourcePath, get.decryptionFlagValues.TempPath) - commons.Printf("skip downloading a data object %s. The file already exists!\n", targetFilePath) - logger.Debugf("skip downloading a data object %s. The file already exists!", targetFilePath) - return nil - } - } - } + decryptedFilename, err := encryptManager.DecryptFilename(sourceFilename) + if err != nil { + return "", "", xerrors.Errorf("failed to decrypt filename %s: %w", sourcePath, err) } - threadsRequired := irodsclient_util.GetNumTasksForParallelTransfer(sourceEntry.Size) - parallelJobManager.Schedule(sourcePath, getTask, threadsRequired, progress.UnitsBytes) - logger.Debugf("scheduled a data object download %s to %s", sourcePath, targetFilePath) - } else { - // dir - logger.Debugf("downloading a collection %s to %s", sourcePath, targetPath) + targetFilePath := commons.MakeTargetLocalFilePath(decryptedFilename, targetPath) - entries, err := filesystem.List(sourceEntry.Path) - if err != nil { - return xerrors.Errorf("failed to list dir %s: %w", sourceEntry.Path, err) - } + return tempFilePath, targetFilePath, nil + } - for _, entry := range entries { - decryptionFlagValuesCopy := decryptionFlagValues + targetFilePath := commons.MakeTargetLocalFilePath(sourcePath, targetPath) - targetDirPath := targetPath - if entry.Type != irodsclient_fs.FileEntry { - // dir - targetDirPath = commons.MakeTargetLocalFilePath(entry.Path, targetPath) - err = os.MkdirAll(targetDirPath, 0766) - if err != nil { - return xerrors.Errorf("failed to make dir %s: %w", targetDirPath, err) - } - } + return "", targetFilePath, nil +} - commons.MarkPathMap(inputPathMap, targetDirPath) +func (get *GetCommand) decryptFile(sourcePath string, encryptedFilePath string, targetPath string) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "GetCommand", + "function": "decryptFile", + }) - err = getOne(parallelJobManager, transferReportManager, inputPathMap, entry.Path, targetDirPath, forceFlagValues, parallelTransferFlagValues, differentialTransferFlagValues, checksumFlagValues, decryptionFlagValuesCopy, postTransferFlagValues) - if err != nil { - return xerrors.Errorf("failed to perform get %s to %s: %w", entry.Path, targetDirPath, err) - } + encryptionMode := commons.DetectEncryptionMode(sourcePath) + encryptManager := get.getEncryptionManagerForDecryption(encryptionMode) + + if get.requireDecryption(sourcePath) { + logger.Debugf("decrypt a data object %s to %s", encryptedFilePath, targetPath) + err := encryptManager.DecryptFile(encryptedFilePath, targetPath) + if err != nil { + return xerrors.Errorf("failed to decrypt %s: %w", encryptedFilePath, err) } + + logger.Debugf("removing a temp file %s", encryptedFilePath) + os.Remove(encryptedFilePath) } + return nil } -func makeGetTargetDirPath(filesystem *irodsclient_fs.FileSystem, sourcePath string, targetPath string, noRoot bool) (string, error) { - cwd := commons.GetCWD() - home := commons.GetHomeDir() - zone := commons.GetZone() - sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) - targetPath = commons.MakeLocalPath(targetPath) - - sourceEntry, err := filesystem.Stat(sourcePath) - if err != nil { - return "", xerrors.Errorf("failed to stat %s: %w", sourcePath, err) - } +func (get *GetCommand) getTask(sourcePath string, targetPath string, sourceSize int64) func(job *commons.ParallelJob) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "GetCommand", + "function": "getTask", + }) - if sourceEntry.Type == irodsclient_fs.FileEntry { - // file - targetFilePath := commons.MakeTargetLocalFilePath(sourcePath, targetPath) - targetDirPath := commons.GetDir(targetFilePath) - _, err := os.Stat(targetDirPath) - if err != nil { - if os.IsNotExist(err) { - return "", irodsclient_types.NewFileNotFoundError(targetDirPath) - } + return func(job *commons.ParallelJob) error { + manager := job.GetManager() + fs := manager.GetFilesystem() - return "", xerrors.Errorf("failed to stat dir %s: %w", targetDirPath, err) + callbackGet := func(processed int64, total int64) { + job.Progress(processed, total, false) } - return targetDirPath, nil - } else { - // dir - _, err := os.Stat(targetPath) - if err != nil { - if os.IsNotExist(err) { - return "", irodsclient_types.NewFileNotFoundError(targetPath) + job.Progress(0, sourceSize, false) + + logger.Debugf("downloading a data object %s to %s", sourcePath, targetPath) + + var downloadErr error + var downloadResult *irodsclient_fs.FileTransferResult + notes := []string{} + + // determine how to download + if get.parallelTransferFlagValues.SingleTread || get.parallelTransferFlagValues.ThreadNumber == 1 { + downloadResult, downloadErr = fs.DownloadFileResumable(sourcePath, "", targetPath, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "icat") + notes = append(notes, "single-thread") + } else if get.parallelTransferFlagValues.RedirectToResource { + downloadResult, downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "redirect-to-resource") + } else if get.parallelTransferFlagValues.Icat { + downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "icat") + notes = append(notes, "multi-thread") + } else { + // auto + if sourceSize >= commons.RedirectToResourceMinSize { + // redirect-to-resource + downloadResult, downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "redirect-to-resource") + } else { + downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "icat") + notes = append(notes, "multi-thread") } - - return "", xerrors.Errorf("failed to stat dir %s: %w", targetPath, err) } - targetDirPath := targetPath + get.transferReportManager.AddTransfer(downloadResult, commons.TransferMethodGet, downloadErr, notes) - if !noRoot { - // make target dir - targetDirPath = commons.MakeTargetLocalFilePath(sourceEntry.Path, targetDirPath) - err = os.MkdirAll(targetDirPath, 0766) - if err != nil { - return "", xerrors.Errorf("failed to make dir %s: %w", targetDirPath, err) - } + if downloadErr != nil { + job.Progress(-1, sourceSize, true) + return xerrors.Errorf("failed to download %s to %s: %w", sourcePath, targetPath, downloadErr) } - return targetDirPath, nil + logger.Debugf("downloaded a data object %s to %s", sourcePath, targetPath) + job.Progress(sourceSize, sourceSize, false) + + return nil } } -func getDeleteExtra(inputPathMap map[string]bool, targetPath string) error { - targetPath = commons.MakeLocalPath(targetPath) +func (get *GetCommand) deleteOnSuccess(sourcePath string) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "GetCommand", + "function": "deleteOnSuccess", + }) + + if get.postTransferFlagValues.DeleteOnSuccess { + logger.Debugf("removing source file %s", sourcePath) + get.filesystem.RemoveFile(sourcePath, true) + } - return getDeleteExtraInternal(inputPathMap, targetPath) + return nil } -func getDeleteExtraInternal(inputPathMap map[string]bool, targetPath string) error { +func (get *GetCommand) deleteExtra() error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "getDeleteExtraInternal", + "struct": "GetCommand", + "function": "deleteExtra", + }) + + if get.syncFlagValues.Delete { + logger.Infof("deleting extra files and dirs under %s", get.targetPath) + targetPath := commons.MakeLocalPath(get.targetPath) + + err := get.deleteExtraInternal(targetPath) + if err != nil { + return xerrors.Errorf("failed to delete extra files: %w", err) + } + } + + return nil +} + +func (get *GetCommand) deleteExtraInternal(targetPath string) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "GetCommand", + "function": "deleteExtraInternal", }) realTargetPath, err := commons.ResolveSymlink(targetPath) @@ -535,8 +645,6 @@ func getDeleteExtraInternal(inputPathMap map[string]bool, targetPath string) err return xerrors.Errorf("failed to resolve symlink %s: %w", targetPath, err) } - logger.Debugf("path %s ==> %s", targetPath, realTargetPath) - targetStat, err := os.Stat(realTargetPath) if err != nil { if os.IsNotExist(err) { @@ -546,22 +654,27 @@ func getDeleteExtraInternal(inputPathMap map[string]bool, targetPath string) err return xerrors.Errorf("failed to stat %s: %w", realTargetPath, err) } - if !targetStat.IsDir() { - // file - if _, ok := inputPathMap[targetPath]; !ok { - // extra file - logger.Debugf("removing an extra file %s", targetPath) - removeErr := os.Remove(targetPath) - if removeErr != nil { - return removeErr - } - } - } else { + // if target is dir, recursive + if targetStat.IsDir() { // dir - if _, ok := inputPathMap[targetPath]; !ok { + if _, ok := get.inputPathMap[targetPath]; !ok { // extra dir logger.Debugf("removing an extra dir %s", targetPath) removeErr := os.RemoveAll(targetPath) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + LocalPath: targetPath, + LocalSize: targetStat.Size(), + Error: removeErr, + Notes: []string{"deleted", "extra"}, + } + + get.transferReportManager.AddFile(reportFile) + if removeErr != nil { return removeErr } @@ -574,12 +687,38 @@ func getDeleteExtraInternal(inputPathMap map[string]bool, targetPath string) err for _, entryInDir := range entries { newTargetPath := commons.MakeTargetLocalFilePath(entryInDir.Name(), targetPath) - err = getDeleteExtraInternal(inputPathMap, newTargetPath) + err = get.deleteExtraInternal(newTargetPath) if err != nil { return err } } } + + return nil + } + + // file + if _, ok := get.inputPathMap[targetPath]; !ok { + // extra file + logger.Debugf("removing an extra file %s", targetPath) + removeErr := os.Remove(targetPath) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + LocalPath: targetPath, + LocalSize: targetStat.Size(), + Error: removeErr, + Notes: []string{"deleted", "extra"}, + } + + get.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } } return nil diff --git a/cmd/subcmd/init.go b/cmd/subcmd/init.go index 2c6bcff..bb9b936 100644 --- a/cmd/subcmd/init.go +++ b/cmd/subcmd/init.go @@ -6,6 +6,7 @@ import ( "github.com/spf13/cobra" "golang.org/x/xerrors" + irodsclient_icommands "github.com/cyverse/go-irodsclient/icommands" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" ) @@ -29,7 +30,35 @@ func AddInitCommand(rootCmd *cobra.Command) { } func processInitCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + init, err := NewInitCommand(command, args) + if err != nil { + return err + } + + return init.Process() +} + +type InitCommand struct { + command *cobra.Command + + initFlagValues *flag.InitFlagValues + + environmentManager *irodsclient_icommands.ICommandsEnvironmentManager + account *irodsclient_types.IRODSAccount +} + +func NewInitCommand(command *cobra.Command, args []string) (*InitCommand, error) { + init := &InitCommand{ + command: command, + + initFlagValues: flag.GetInitFlagValues(), + } + + return init, nil +} + +func (init *InitCommand) Process() error { + cont, err := flag.ProcessCommonFlags(init.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -38,9 +67,7 @@ func processInitCommand(command *cobra.Command, args []string) error { return nil } - initFlagValues := flag.GetInitFlagValues() - - environmentManager := commons.GetEnvironmentManager() + init.environmentManager = commons.GetEnvironmentManager() // handle local flags updated, err := commons.ReinputFields() @@ -48,25 +75,24 @@ func processInitCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input fields: %w", err) } - account, err := environmentManager.ToIRODSAccount() + init.account, err = init.environmentManager.ToIRODSAccount() if err != nil { return xerrors.Errorf("failed to get iRODS account info from iCommands Environment: %w", err) } // update PAM TTL - account.PamTTL = initFlagValues.PamTTL + init.account.PamTTL = init.initFlagValues.PamTTL // test connect - conn, err := commons.GetIRODSConnection(account) + conn, err := commons.GetIRODSConnection(init.account) if err != nil { return xerrors.Errorf("failed to connect to iRODS server: %w", err) } - defer conn.Disconnect() - if account.AuthenticationScheme == irodsclient_types.AuthSchemePAM { - // set pam token - environmentManager.PamToken = conn.GetPAMToken() + if init.account.AuthenticationScheme == irodsclient_types.AuthSchemePAM { + // update pam token + init.environmentManager.PamToken = conn.GetPAMToken() } if updated { @@ -82,5 +108,6 @@ func processInitCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to print account info: %w", err) } } + return nil } diff --git a/cmd/subcmd/ls.go b/cmd/subcmd/ls.go index 77b2ee4..b0ff0f5 100644 --- a/cmd/subcmd/ls.go +++ b/cmd/subcmd/ls.go @@ -45,12 +45,54 @@ func AddLsCommand(rootCmd *cobra.Command) { } func processLsCommand(command *cobra.Command, args []string) error { + ls, err := NewLsCommand(command, args) + if err != nil { + return err + } + + return ls.Process() +} + +type LsCommand struct { + command *cobra.Command + + ticketAccessFlagValues *flag.TicketAccessFlagValues + listFlagValues *flag.ListFlagValues + decryptionFlagValues *flag.DecryptionFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + sourcePaths []string +} + +func NewLsCommand(command *cobra.Command, args []string) (*LsCommand, error) { + ls := &LsCommand{ + command: command, + + ticketAccessFlagValues: flag.GetTicketAccessFlagValues(), + listFlagValues: flag.GetListFlagValues(), + decryptionFlagValues: flag.GetDecryptionFlagValues(command), + } + + // path + ls.sourcePaths = args[:] + + if len(args) == 0 { + ls.sourcePaths = []string{"."} + } + + return ls, nil +} + +func (ls *LsCommand) Process() error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "processLsCommand", + "struct": "LsCommand", + "function": "Process", }) - cont, err := flag.ProcessCommonFlags(command) + cont, err := flag.ProcessCommonFlags(ls.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -65,16 +107,12 @@ func processLsCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - ticketAccessFlagValues := flag.GetTicketAccessFlagValues() - listFlagValues := flag.GetListFlagValues() - decryptionFlagValues := flag.GetDecryptionFlagValues(command) - + // config appConfig := commons.GetConfig() - syncAccount := false - if len(ticketAccessFlagValues.Name) > 0 { - logger.Debugf("use ticket: %s", ticketAccessFlagValues.Name) - appConfig.Ticket = ticketAccessFlagValues.Name + if len(ls.ticketAccessFlagValues.Name) > 0 { + logger.Debugf("use ticket: %s", ls.ticketAccessFlagValues.Name) + appConfig.Ticket = ls.ticketAccessFlagValues.Name syncAccount = true } @@ -86,26 +124,21 @@ func processLsCommand(command *cobra.Command, args []string) error { } // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + ls.account = commons.GetAccount() + ls.filesystem, err = commons.GetIRODSFSClient(ls.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } - defer filesystem.Release() + defer ls.filesystem.Release() // set default key for decryption - if len(decryptionFlagValues.Key) == 0 { - decryptionFlagValues.Key = account.Password + if len(ls.decryptionFlagValues.Key) == 0 { + ls.decryptionFlagValues.Key = ls.account.Password } - sourcePaths := args[:] - - if len(args) == 0 { - sourcePaths = []string{"."} - } - - for _, sourcePath := range sourcePaths { - err = listOne(filesystem, sourcePath, listFlagValues, decryptionFlagValues) + // run + for _, sourcePath := range ls.sourcePaths { + err = ls.listOne(sourcePath, false) if err != nil { return xerrors.Errorf("failed to perform ls %s: %w", sourcePath, err) } @@ -114,36 +147,43 @@ func processLsCommand(command *cobra.Command, args []string) error { return nil } -func listOne(fs *irodsclient_fs.FileSystem, sourcePath string, listFlagValues *flag.ListFlagValues, decryptionFlagValues *flag.DecryptionFlagValues) error { +func (ls *LsCommand) requireDecryptionByMeta(sourceEntry *irodsclient_fs.Entry) bool { + // load encryption config from meta + if !ls.decryptionFlagValues.NoDecryption && !ls.decryptionFlagValues.IgnoreMeta { + sourceDir := sourceEntry.Path + if !sourceEntry.IsDir() { + sourceDir = commons.GetDir(sourceEntry.Path) + } + + encryptionConfig := commons.GetEncryptionConfigFromMeta(ls.filesystem, sourceDir) + + return encryptionConfig.Required + } + + return false +} + +func (ls *LsCommand) listOne(sourcePath string, requireDecryption bool) error { cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) - sourceEntry, err := fs.Stat(sourcePath) + sourceEntry, err := ls.filesystem.Stat(sourcePath) if err != nil { return xerrors.Errorf("failed to stat %s: %w", sourcePath, err) } // load encryption config from meta - if !decryptionFlagValues.NoDecryption && !decryptionFlagValues.IgnoreMeta { - sourceDir := sourcePath - if !sourceEntry.IsDir() { - sourceDir = commons.GetDir(sourcePath) - } - - encryptionConfig := commons.GetEncryptionConfigFromMeta(fs, sourceDir) - - if encryptionConfig.Required { - decryptionFlagValues.Decryption = encryptionConfig.Required - } + if !requireDecryption { + requireDecryption = ls.requireDecryptionByMeta(sourceEntry) } - connection, err := fs.GetMetadataConnection() + connection, err := ls.filesystem.GetMetadataConnection() if err != nil { return xerrors.Errorf("failed to get connection: %w", err) } - defer fs.ReturnMetadataConnection(connection) + defer ls.filesystem.ReturnMetadataConnection(connection) collection, err := irodsclient_irodsfs.GetCollection(connection, sourcePath) if err != nil { @@ -153,6 +193,7 @@ func listOne(fs *irodsclient_fs.FileSystem, sourcePath string, listFlagValues *f } if err == nil { + // collection colls, err := irodsclient_irodsfs.ListSubCollections(connection, sourcePath) if err != nil { return xerrors.Errorf("failed to list sub-collections in %s: %w", sourcePath, err) @@ -163,8 +204,8 @@ func listOne(fs *irodsclient_fs.FileSystem, sourcePath string, listFlagValues *f return xerrors.Errorf("failed to list data-objects in %s: %w", sourcePath, err) } - printDataObjects(objs, listFlagValues, decryptionFlagValues) - printCollections(colls, listFlagValues) + ls.printDataObjects(objs, requireDecryption) + ls.printCollections(colls) return nil } @@ -182,22 +223,46 @@ func listOne(fs *irodsclient_fs.FileSystem, sourcePath string, listFlagValues *f } entries := []*irodsclient_types.IRODSDataObject{entry} - printDataObjects(entries, listFlagValues, decryptionFlagValues) + ls.printDataObjects(entries, requireDecryption) + return nil } -func flattenReplicas(objects []*irodsclient_types.IRODSDataObject) []*FlatReplica { +func (ls *LsCommand) printCollections(entries []*irodsclient_types.IRODSCollection) { + sort.SliceStable(entries, ls.getCollectionSortFunction(entries, ls.listFlagValues.SortOrder, ls.listFlagValues.SortReverse)) + for _, entry := range entries { + fmt.Printf(" C- %s\n", entry.Path) + } +} + +func (ls *LsCommand) printDataObjects(entries []*irodsclient_types.IRODSDataObject, requireDecryption bool) { + if ls.listFlagValues.Format == commons.ListFormatNormal { + sort.SliceStable(entries, ls.getDataObjectSortFunction(entries, ls.listFlagValues.SortOrder, ls.listFlagValues.SortReverse)) + for _, entry := range entries { + ls.printDataObjectShort(entry, requireDecryption) + } + } else { + replicas := ls.flattenReplicas(entries) + sort.SliceStable(replicas, ls.getFlatReplicaSortFunction(replicas, ls.listFlagValues.SortOrder, ls.listFlagValues.SortReverse)) + ls.printReplicas(replicas, requireDecryption) + } +} + +func (ls *LsCommand) flattenReplicas(objects []*irodsclient_types.IRODSDataObject) []*FlatReplica { var result []*FlatReplica for _, object := range objects { for _, replica := range object.Replicas { - flatReplica := FlatReplica{DataObject: object, Replica: replica} + flatReplica := FlatReplica{ + DataObject: object, + Replica: replica, + } result = append(result, &flatReplica) } } return result } -func getFlatReplicaSortFunction(entries []*FlatReplica, sortOrder commons.ListSortOrder, sortReverse bool) func(i int, j int) bool { +func (ls *LsCommand) getFlatReplicaSortFunction(entries []*FlatReplica, sortOrder commons.ListSortOrder, sortReverse bool) func(i int, j int) bool { if sortReverse { switch sortOrder { case commons.ListSortOrderName: @@ -259,20 +324,7 @@ func getFlatReplicaSortFunction(entries []*FlatReplica, sortOrder commons.ListSo } } -func printDataObjects(entries []*irodsclient_types.IRODSDataObject, listFlagValues *flag.ListFlagValues, decryptionFlagValues *flag.DecryptionFlagValues) { - if listFlagValues.Format == commons.ListFormatNormal { - sort.SliceStable(entries, getDataObjectSortFunction(entries, listFlagValues.SortOrder, listFlagValues.SortReverse)) - for _, entry := range entries { - printDataObjectShort(entry, decryptionFlagValues) - } - } else { - replicas := flattenReplicas(entries) - sort.SliceStable(replicas, getFlatReplicaSortFunction(replicas, listFlagValues.SortOrder, listFlagValues.SortReverse)) - printReplicas(replicas, listFlagValues, decryptionFlagValues) - } -} - -func getDataObjectSortFunction(entries []*irodsclient_types.IRODSDataObject, sortOrder commons.ListSortOrder, sortReverse bool) func(i int, j int) bool { +func (ls *LsCommand) getDataObjectSortFunction(entries []*irodsclient_types.IRODSDataObject, sortOrder commons.ListSortOrder, sortReverse bool) func(i int, j int) bool { if sortReverse { switch sortOrder { case commons.ListSortOrderName: @@ -287,8 +339,8 @@ func getDataObjectSortFunction(entries []*irodsclient_types.IRODSDataObject, sor } case commons.ListSortOrderTime: return func(i int, j int) bool { - return (getDataObjectModifyTime(entries[i]).After(getDataObjectModifyTime(entries[j]))) || - (getDataObjectModifyTime(entries[i]).Equal(getDataObjectModifyTime(entries[j])) && + return (ls.getDataObjectModifyTime(entries[i]).After(ls.getDataObjectModifyTime(entries[j]))) || + (ls.getDataObjectModifyTime(entries[i]).Equal(ls.getDataObjectModifyTime(entries[j])) && entries[i].Name < entries[j].Name) } case commons.ListSortOrderSize: @@ -317,8 +369,8 @@ func getDataObjectSortFunction(entries []*irodsclient_types.IRODSDataObject, sor } case commons.ListSortOrderTime: return func(i int, j int) bool { - return (getDataObjectModifyTime(entries[i]).Before(getDataObjectModifyTime(entries[j]))) || - (getDataObjectModifyTime(entries[i]).Equal(getDataObjectModifyTime(entries[j])) && + return (ls.getDataObjectModifyTime(entries[i]).Before(ls.getDataObjectModifyTime(entries[j]))) || + (ls.getDataObjectModifyTime(entries[i]).Equal(ls.getDataObjectModifyTime(entries[j])) && entries[i].Name < entries[j].Name) } case commons.ListSortOrderSize: @@ -334,7 +386,7 @@ func getDataObjectSortFunction(entries []*irodsclient_types.IRODSDataObject, sor } } -func getDataObjectModifyTime(object *irodsclient_types.IRODSDataObject) time.Time { +func (ls *LsCommand) getDataObjectModifyTime(object *irodsclient_types.IRODSDataObject) time.Time { // ModifyTime of data object is considered to be ModifyTime of replica modified most recently maxTime := object.Replicas[0].ModifyTime for _, t := range object.Replicas[1:] { @@ -345,19 +397,21 @@ func getDataObjectModifyTime(object *irodsclient_types.IRODSDataObject) time.Tim return maxTime } -func printDataObjectShort(entry *irodsclient_types.IRODSDataObject, decryptionFlagValues *flag.DecryptionFlagValues) { +func (ls *LsCommand) printDataObjectShort(entry *irodsclient_types.IRODSDataObject, requireDecryption bool) { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "LsCommand", "function": "printDataObjectShort", }) newName := entry.Name - if decryptionFlagValues.Decryption { - // need to decrypted + if requireDecryption { + // need to decrypt encryptionMode := commons.DetectEncryptionMode(newName) if encryptionMode != commons.EncryptionModeUnknown { - encryptManager := getEncryptionManagerForDecrypt(encryptionMode, decryptionFlagValues) + encryptManager := ls.getEncryptionManagerForDecryption(encryptionMode) + decryptedFilename, err := encryptManager.DecryptFilename(newName) if err != nil { logger.Debugf("%+v", err) @@ -371,25 +425,40 @@ func printDataObjectShort(entry *irodsclient_types.IRODSDataObject, decryptionFl fmt.Printf(" %s\n", newName) } -func printReplicas(flatReplicas []*FlatReplica, listFlagValues *flag.ListFlagValues, decryptionFlagValues *flag.DecryptionFlagValues) { +func (ls *LsCommand) printReplicas(flatReplicas []*FlatReplica, requireDecryption bool) { for _, flatReplica := range flatReplicas { - printReplica(*flatReplica, listFlagValues, decryptionFlagValues) + ls.printReplica(*flatReplica, requireDecryption) } } -func printReplica(flatReplica FlatReplica, listFlagValues *flag.ListFlagValues, decryptionFlagValues *flag.DecryptionFlagValues) { +func (ls *LsCommand) getEncryptionManagerForDecryption(mode commons.EncryptionMode) *commons.EncryptionManager { + manager := commons.NewEncryptionManager(mode) + + switch mode { + case commons.EncryptionModeWinSCP, commons.EncryptionModePGP: + manager.SetKey([]byte(ls.decryptionFlagValues.Key)) + case commons.EncryptionModeSSH: + manager.SetPublicPrivateKey(ls.decryptionFlagValues.PrivateKeyPath) + } + + return manager +} + +func (ls *LsCommand) printReplica(flatReplica FlatReplica, requireDecryption bool) { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "LsCommand", "function": "printReplica", }) newName := flatReplica.DataObject.Name - if decryptionFlagValues.Decryption { - // need to decrypted + if requireDecryption { + // need to decrypt encryptionMode := commons.DetectEncryptionMode(newName) if encryptionMode != commons.EncryptionModeUnknown { - encryptManager := getEncryptionManagerForDecrypt(encryptionMode, decryptionFlagValues) + encryptManager := ls.getEncryptionManagerForDecryption(encryptionMode) + decryptedFilename, err := encryptManager.DecryptFilename(newName) if err != nil { logger.Debugf("%+v", err) @@ -401,35 +470,28 @@ func printReplica(flatReplica FlatReplica, listFlagValues *flag.ListFlagValues, } size := fmt.Sprintf("%v", flatReplica.DataObject.Size) - if listFlagValues.HumanReadableSizes { + if ls.listFlagValues.HumanReadableSizes { size = humanize.Bytes(uint64(flatReplica.DataObject.Size)) } - switch listFlagValues.Format { + switch ls.listFlagValues.Format { case commons.ListFormatNormal: fmt.Printf(" %d\t%s\n", flatReplica.Replica.Number, newName) case commons.ListFormatLong: modTime := commons.MakeDateTimeString(flatReplica.Replica.ModifyTime) fmt.Printf(" %s\t%d\t%s\t%s\t%s\t%s\t%s\n", flatReplica.Replica.Owner, flatReplica.Replica.Number, flatReplica.Replica.ResourceHierarchy, - size, modTime, getStatusMark(flatReplica.Replica.Status), newName) + size, modTime, ls.getStatusMark(flatReplica.Replica.Status), newName) case commons.ListFormatVeryLong: modTime := commons.MakeDateTimeString(flatReplica.Replica.ModifyTime) fmt.Printf(" %s\t%d\t%s\t%s\t%s\t%s\t%s\n", flatReplica.Replica.Owner, flatReplica.Replica.Number, flatReplica.Replica.ResourceHierarchy, - size, modTime, getStatusMark(flatReplica.Replica.Status), newName) + size, modTime, ls.getStatusMark(flatReplica.Replica.Status), newName) fmt.Printf(" %s\t%s\n", flatReplica.Replica.Checksum.IRODSChecksumString, flatReplica.Replica.Path) default: fmt.Printf(" %d\t%s\n", flatReplica.Replica.Number, newName) } } -func printCollections(entries []*irodsclient_types.IRODSCollection, listFlagValues *flag.ListFlagValues) { - sort.SliceStable(entries, getCollectionSortFunction(entries, listFlagValues.SortOrder, listFlagValues.SortReverse)) - for _, entry := range entries { - fmt.Printf(" C- %s\n", entry.Path) - } -} - -func getCollectionSortFunction(entries []*irodsclient_types.IRODSCollection, sortOrder commons.ListSortOrder, sortReverse bool) func(i int, j int) bool { +func (ls *LsCommand) getCollectionSortFunction(entries []*irodsclient_types.IRODSCollection, sortOrder commons.ListSortOrder, sortReverse bool) func(i int, j int) bool { if sortReverse { switch sortOrder { case commons.ListSortOrderName: @@ -470,7 +532,7 @@ func getCollectionSortFunction(entries []*irodsclient_types.IRODSCollection, sor } } -func getStatusMark(status string) string { +func (ls *LsCommand) getStatusMark(status string) string { switch status { case "0": return "X" // stale diff --git a/cmd/subcmd/put.go b/cmd/subcmd/put.go index a478315..f1e85a8 100644 --- a/cmd/subcmd/put.go +++ b/cmd/subcmd/put.go @@ -293,26 +293,26 @@ func putOne(parallelJobManager *commons.ParallelJobManager, inputPathMap map[str // determine how to download if parallelTransferFlagValues.SingleTread || parallelTransferFlagValues.ThreadNumber == 1 { - uploadErr = fs.UploadFile(sourcePath, targetFilePath, "", false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) + _, uploadErr = fs.UploadFile(sourcePath, targetFilePath, "", false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) } else if parallelTransferFlagValues.RedirectToResource { - uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) + _, uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) } else if parallelTransferFlagValues.Icat { - uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) + _, uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) } else { // auto if sourceStat.Size() >= commons.RedirectToResourceMinSize { // redirect-to-resource - uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) + _, uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) } else { if filesystem.SupportParallelUpload() { - uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) + _, uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) } else { if sourceStat.Size() >= commons.ParallelUploadMinSize { // does not support parall upload via iCAT // redirect-to-resource - uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) + _, uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) } else { - uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) + _, uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) } } } diff --git a/commons/bundle_transfer.go b/commons/bundle_transfer.go index 16071e0..2a0c989 100644 --- a/commons/bundle_transfer.go +++ b/commons/bundle_transfer.go @@ -200,8 +200,8 @@ type BundleTransferManager struct { scheduleWait sync.WaitGroup transferWait sync.WaitGroup - bundlesScheduledCounter atomic.Uint64 - bundlesDoneCounter atomic.Uint64 + bundlesScheduledCounter int64 + bundlesDoneCounter int64 } // NewBundleTransferManager creates a new BundleTransferManager @@ -236,8 +236,8 @@ func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, irodsDestPath strin scheduleWait: sync.WaitGroup{}, transferWait: sync.WaitGroup{}, - bundlesScheduledCounter: atomic.Uint64{}, - bundlesDoneCounter: atomic.Uint64{}, + bundlesScheduledCounter: 0, + bundlesDoneCounter: 0, } if manager.uploadThreadNum > UploadTreadNumMax { @@ -301,7 +301,7 @@ func (manager *BundleTransferManager) Schedule(source string, dir bool, size int manager.mutex.Lock() manager.currentBundle = nil manager.transferWait.Add(1) - manager.bundlesScheduledCounter.Add(1) + atomic.AddInt64(&manager.bundlesScheduledCounter, 1) } } @@ -400,7 +400,7 @@ func (manager *BundleTransferManager) DoneScheduling() { manager.bundles = append(manager.bundles, manager.currentBundle) manager.currentBundle = nil manager.transferWait.Add(1) - manager.bundlesScheduledCounter.Add(1) + atomic.AddInt64(&manager.bundlesScheduledCounter, 1) } manager.mutex.Unlock() @@ -437,8 +437,8 @@ func (manager *BundleTransferManager) Wait() error { return manager.lastError } - if manager.bundlesDoneCounter.Load() != manager.bundlesScheduledCounter.Load() { - return xerrors.Errorf("bundles '%d/%d' were canceled!", manager.bundlesDoneCounter.Load(), manager.bundlesScheduledCounter.Load()) + if manager.bundlesDoneCounter != manager.bundlesScheduledCounter { + return xerrors.Errorf("bundles '%d/%d' were canceled!", manager.bundlesDoneCounter, manager.bundlesScheduledCounter) } return nil @@ -1071,26 +1071,26 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error // determine how to download if manager.singleThreaded || manager.uploadThreadNum == 1 { - err = manager.filesystem.UploadFile(bundle.localBundlePath, bundle.irodsBundlePath, "", false, false, false, callback) + _, err = manager.filesystem.UploadFile(bundle.localBundlePath, bundle.irodsBundlePath, "", false, true, true, callback) } else if manager.redirectToResource { - err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) + _, err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, true, true, callback) } else if manager.useIcat { - err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) + _, err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, true, true, callback) } else { // auto if bundle.size >= RedirectToResourceMinSize { // redirect-to-resource - err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) + _, err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, true, true, callback) } else { if manager.filesystem.SupportParallelUpload() { - err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) + _, err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) } else { if bundle.size >= ParallelUploadMinSize { // does not support parall upload via iCAT // redirect-to-resource - err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) + _, err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) } else { - err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) + _, err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) } } } @@ -1166,26 +1166,26 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error } else { // determine how to download if manager.singleThreaded || manager.uploadThreadNum == 1 { - err = manager.filesystem.UploadFile(file.LocalPath, file.IRODSPath, "", false, false, false, callbackFileUpload) + _, err = manager.filesystem.UploadFile(file.LocalPath, file.IRODSPath, "", false, false, false, callbackFileUpload) } else if manager.redirectToResource { - err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) + _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) } else if manager.useIcat { - err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) + _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) } else { // auto if bundle.size >= RedirectToResourceMinSize { // redirect-to-resource - err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) + _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) } else { if manager.filesystem.SupportParallelUpload() { - err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) + _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) } else { if bundle.size >= ParallelUploadMinSize { // does not support parall upload via iCAT // redirect-to-resource - err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) + _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) } else { - err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) + _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) } } } @@ -1254,7 +1254,7 @@ func (manager *BundleTransferManager) processBundleExtract(bundle *Bundle) error // set it done bundle.Done() - manager.bundlesDoneCounter.Add(1) + atomic.AddInt64(&manager.bundlesDoneCounter, 1) logger.Debugf("extracted bundle %d at %s to %s", bundle.index, bundle.irodsBundlePath, manager.irodsDestPath) return nil diff --git a/commons/parallel.go b/commons/parallel.go index b8b192b..4c3d4b6 100644 --- a/commons/parallel.go +++ b/commons/parallel.go @@ -68,8 +68,8 @@ type ParallelJobManager struct { scheduleWait sync.WaitGroup jobWait sync.WaitGroup - jobsScheduledCounter atomic.Uint64 - jobsDoneCounter atomic.Uint64 + jobsScheduledCounter int64 + jobsDoneCounter int64 } // NewParallelJobManager creates a new ParallelJobManager @@ -89,8 +89,8 @@ func NewParallelJobManager(fs *irodsclient_fs.FileSystem, maxThreads int, showPr scheduleWait: sync.WaitGroup{}, jobWait: sync.WaitGroup{}, - jobsScheduledCounter: atomic.Uint64{}, - jobsDoneCounter: atomic.Uint64{}, + jobsScheduledCounter: 0, + jobsDoneCounter: 0, } manager.availableThreadWaitCondition = sync.NewCond(&manager.mutex) @@ -132,7 +132,7 @@ func (manager *ParallelJobManager) Schedule(name string, task ParallelJobTask, t manager.pendingJobs <- job manager.jobWait.Add(1) - manager.jobsScheduledCounter.Add(1) + atomic.AddInt64(&manager.jobsScheduledCounter, 1) return nil } @@ -161,8 +161,8 @@ func (manager *ParallelJobManager) Wait() error { return manager.lastError } - if manager.jobsDoneCounter.Load() != manager.jobsScheduledCounter.Load() { - return xerrors.Errorf("jobs '%d/%d' were canceled!", manager.jobsDoneCounter.Load(), manager.jobsScheduledCounter.Load()) + if manager.jobsDoneCounter != manager.jobsScheduledCounter { + return xerrors.Errorf("jobs '%d/%d' were canceled!", manager.jobsDoneCounter, manager.jobsScheduledCounter) } return nil @@ -297,7 +297,7 @@ func (manager *ParallelJobManager) Start() { manager.jobWait.Done() if pjob.done { // increase jobs done counter - manager.jobsDoneCounter.Add(1) + atomic.AddInt64(&manager.jobsDoneCounter, 1) } manager.mutex.Lock() diff --git a/commons/transfer_report.go b/commons/transfer_report.go index a924ae8..ec789ab 100644 --- a/commons/transfer_report.go +++ b/commons/transfer_report.go @@ -7,6 +7,7 @@ import ( "io" "os" "strings" + "sync" "time" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" @@ -82,6 +83,7 @@ type TransferReportManager struct { reportToStdout bool writer io.WriteCloser + lock sync.Mutex } // NewTransferReportManager creates a new TransferReportManager @@ -107,6 +109,7 @@ func NewTransferReportManager(report bool, reportPath string, reportToStdout boo reportToStdout: reportToStdout, writer: writer, + lock: sync.Mutex{}, } return manager, nil @@ -133,6 +136,9 @@ func (manager *TransferReportManager) AddFile(file *TransferReportFile) error { return nil } + manager.lock.Lock() + defer manager.lock.Unlock() + lineOutput := "" if manager.reportToStdout { // line print diff --git a/go.mod b/go.mod index 06f1a43..4823202 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/creativeprojects/go-selfupdate v1.0.1 - github.com/cyverse/go-irodsclient v0.14.12 + github.com/cyverse/go-irodsclient v0.14.13 github.com/dustin/go-humanize v1.0.1 github.com/gliderlabs/ssh v0.3.5 github.com/jedib0t/go-pretty/v6 v6.3.1 diff --git a/go.sum b/go.sum index 87ad869..b31cc64 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/creativeprojects/go-selfupdate v1.0.1 h1:5Un4MTv4puCR5GBgkDLC14J72flj github.com/creativeprojects/go-selfupdate v1.0.1/go.mod h1:nm7AWUJfrfYt/SB97NAcMhR0KEpPqlrVHXkWFti+ezw= github.com/cyverse/go-irodsclient v0.14.12 h1:CTrS/pl9ADAuF5De1s4eB2Y8et29SSae7KYW09M8PZc= github.com/cyverse/go-irodsclient v0.14.12/go.mod h1:eBXha3cwfrM0p1ijYVqsrLJQHpRwTfpA4c5dKCQsQFc= +github.com/cyverse/go-irodsclient v0.14.13 h1:PIXwZTajiOXUwmFvZLIVAFQpWsV0CXiixDDYVvdeEYU= +github.com/cyverse/go-irodsclient v0.14.13/go.mod h1:eBXha3cwfrM0p1ijYVqsrLJQHpRwTfpA4c5dKCQsQFc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 6f4294533468fa5f7337fb8e5bdc44a56c8a9ba4 Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Thu, 1 Aug 2024 17:22:31 -0700 Subject: [PATCH 03/16] wip: more refactoring --- cmd/subcmd/bun.go | 6 +- cmd/subcmd/cat.go | 115 +++++++++++++++++++++------------- cmd/subcmd/cd.go | 69 ++++++++++++++------ cmd/subcmd/copy-sftp-id.go | 125 +++++++++++++++++++++++++------------ 4 files changed, 207 insertions(+), 108 deletions(-) diff --git a/cmd/subcmd/bun.go b/cmd/subcmd/bun.go index 080674a..f7a61aa 100644 --- a/cmd/subcmd/bun.go +++ b/cmd/subcmd/bun.go @@ -89,12 +89,12 @@ func (bun *BunCommand) Process() error { } // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + bun.account = commons.GetAccount() + bun.filesystem, err = commons.GetIRODSFSClient(bun.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } - defer filesystem.Release() + defer bun.filesystem.Release() // run for _, sourcePath := range bun.sourcePaths { diff --git a/cmd/subcmd/cat.go b/cmd/subcmd/cat.go index faa503d..a3561e4 100644 --- a/cmd/subcmd/cat.go +++ b/cmd/subcmd/cat.go @@ -5,6 +5,7 @@ import ( "io" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -31,12 +32,46 @@ func AddCatCommand(rootCmd *cobra.Command) { } func processCatCommand(command *cobra.Command, args []string) error { + cat, err := NewCatCommand(command, args) + if err != nil { + return err + } + + return cat.Process() +} + +type CatCommand struct { + command *cobra.Command + + ticketAccessFlagValues *flag.TicketAccessFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + sourcePaths []string +} + +func NewCatCommand(command *cobra.Command, args []string) (*CatCommand, error) { + cat := &CatCommand{ + command: command, + + ticketAccessFlagValues: flag.GetTicketAccessFlagValues(), + } + + // path + cat.sourcePaths = args + + return cat, nil +} + +func (cat *CatCommand) Process() error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "processCatCommand", + "struct": "CatCommand", + "function": "Process", }) - cont, err := flag.ProcessCommonFlags(command) + cont, err := flag.ProcessCommonFlags(cat.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -51,13 +86,12 @@ func processCatCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - ticketAccessFlagValues := flag.GetTicketAccessFlagValues() - + // config appConfig := commons.GetConfig() syncAccount := false - if len(ticketAccessFlagValues.Name) > 0 { - logger.Debugf("use ticket: %s", ticketAccessFlagValues.Name) - appConfig.Ticket = ticketAccessFlagValues.Name + if len(cat.ticketAccessFlagValues.Name) > 0 { + logger.Debugf("use ticket: %s", cat.ticketAccessFlagValues.Name) + appConfig.Ticket = cat.ticketAccessFlagValues.Name syncAccount = true } @@ -69,65 +103,58 @@ func processCatCommand(command *cobra.Command, args []string) error { } // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + cat.account = commons.GetAccount() + cat.filesystem, err = commons.GetIRODSFSClient(cat.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer cat.filesystem.Release() - defer filesystem.Release() - - for _, sourcePath := range args { - err = catOne(filesystem, sourcePath) + // run + for _, sourcePath := range cat.sourcePaths { + err = cat.catOne(sourcePath) if err != nil { return xerrors.Errorf("failed to perform cat %s: %w", sourcePath, err) } } + return nil } -func catOne(filesystem *irodsclient_fs.FileSystem, targetPath string) error { - logger := log.WithFields(log.Fields{ - "package": "subcmd", - "function": "catOne", - }) - +func (cat *CatCommand) catOne(sourcePath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() - targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) + sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) - targetEntry, err := filesystem.Stat(targetPath) + sourceEntry, err := cat.filesystem.Stat(sourcePath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", targetPath, err) + return xerrors.Errorf("failed to stat %s: %w", sourcePath, err) } - if targetEntry.Type == irodsclient_fs.FileEntry { - // file - logger.Debugf("showing the content of a data object %s", targetPath) - fh, err := filesystem.OpenFile(targetPath, "", "r") - if err != nil { - return xerrors.Errorf("failed to open file %s: %w", targetPath, err) - } - - defer fh.Close() + if sourceEntry.Type != irodsclient_fs.FileEntry { + return xerrors.Errorf("cannot show the content of a collection") + } - buf := make([]byte, 10240) // 10KB buffer - for { - readLen, err := fh.Read(buf) - if readLen > 0 { - fmt.Printf("%s", string(buf[:readLen])) - } + // file + fh, err := cat.filesystem.OpenFile(sourcePath, "", "r") + if err != nil { + return xerrors.Errorf("failed to open file %s: %w", sourcePath, err) + } + defer fh.Close() - if err == io.EOF { - // EOF - break - } + buf := make([]byte, 10240) // 10KB buffer + for { + readLen, err := fh.Read(buf) + if readLen > 0 { + fmt.Printf("%s", string(buf[:readLen])) } - } else { - // dir - return xerrors.Errorf("cannot show the content of a collection") + if err == io.EOF { + // EOF + break + } } + return nil } diff --git a/cmd/subcmd/cd.go b/cmd/subcmd/cd.go index ee18e89..291bcf1 100644 --- a/cmd/subcmd/cd.go +++ b/cmd/subcmd/cd.go @@ -3,6 +3,7 @@ package subcmd import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_irodsfs "github.com/cyverse/go-irodsclient/irods/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -27,7 +28,42 @@ func AddCdCommand(rootCmd *cobra.Command) { } func processCdCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + cd, err := NewCdCommand(command, args) + if err != nil { + return err + } + + return cd.Process() +} + +type CdCommand struct { + command *cobra.Command + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + targetPath string +} + +func NewCdCommand(command *cobra.Command, args []string) (*CdCommand, error) { + cd := &CdCommand{ + command: command, + } + + // path + cd.targetPath = "" + if len(args) == 0 { + // move to home dir + cd.targetPath = "~" + } else if len(args) == 1 { + cd.targetPath = args[0] + } + + return cd, nil +} + +func (cd *CdCommand) Process() error { + cont, err := flag.ProcessCommonFlags(cd.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -42,34 +78,27 @@ func processCdCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - // Create a connection - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + // Create a file system + cd.account = commons.GetAccount() + cd.filesystem, err = commons.GetIRODSFSClient(cd.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer cd.filesystem.Release() - defer filesystem.Release() - - targetPath := "" - if len(args) == 0 { - // move to home dir - targetPath = "~" - } else if len(args) == 1 { - targetPath = args[0] - } - - // cd - err = changeWorkingDir(filesystem, targetPath) + // run + err = cd.changeWorkingDir(cd.targetPath) if err != nil { - return xerrors.Errorf("failed to perform cd %s: %w", targetPath, err) + return xerrors.Errorf("failed to perform cd %s: %w", cd.targetPath, err) } + return nil } -func changeWorkingDir(fs *irodsclient_fs.FileSystem, collectionPath string) error { +func (cd *CdCommand) changeWorkingDir(collectionPath string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "CdCommand", "function": "changeWorkingDir", }) @@ -78,11 +107,11 @@ func changeWorkingDir(fs *irodsclient_fs.FileSystem, collectionPath string) erro zone := commons.GetZone() collectionPath = commons.MakeIRODSPath(cwd, home, zone, collectionPath) - connection, err := fs.GetMetadataConnection() + connection, err := cd.filesystem.GetMetadataConnection() if err != nil { return xerrors.Errorf("failed to get connection: %w", err) } - defer fs.ReturnMetadataConnection(connection) + defer cd.filesystem.ReturnMetadataConnection(connection) logger.Debugf("changing working dir: %s", collectionPath) diff --git a/cmd/subcmd/copy-sftp-id.go b/cmd/subcmd/copy-sftp-id.go index 4a9627c..4645651 100644 --- a/cmd/subcmd/copy-sftp-id.go +++ b/cmd/subcmd/copy-sftp-id.go @@ -9,6 +9,7 @@ import ( "strings" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" "github.com/gliderlabs/ssh" @@ -38,7 +39,39 @@ func AddCopySftpIdCommand(rootCmd *cobra.Command) { } func processCopySftpIdCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + copy, err := NewCopySftpIdCommand(command, args) + if err != nil { + return err + } + + return copy.Process() +} + +type CopySftpIdCommand struct { + command *cobra.Command + + forceFlagValues *flag.ForceFlagValues + dryRunFlagValues *flag.DryRunFlagValues + sftpIDFlagValues *flag.SFTPIDFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem +} + +func NewCopySftpIdCommand(command *cobra.Command, args []string) (*CopySftpIdCommand, error) { + copy := &CopySftpIdCommand{ + command: command, + + forceFlagValues: flag.GetForceFlagValues(), + dryRunFlagValues: flag.GetDryRunFlagValues(), + sftpIDFlagValues: flag.GetSFTPIDFlagValues(), + } + + return copy, nil +} + +func (copy *CopySftpIdCommand) Process() error { + cont, err := flag.ProcessCommonFlags(copy.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -53,45 +86,55 @@ func processCopySftpIdCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - forceFlagValues := flag.GetForceFlagValues() - dryRunFlagValues := flag.GetDryRunFlagValues() - sftpIDFlagValues := flag.GetSFTPIDFlagValues() - // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + copy.account = commons.GetAccount() + copy.filesystem, err = commons.GetIRODSFSClient(copy.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer copy.filesystem.Release() - defer filesystem.Release() - + // run // search identity files to be copied - identityFiles := []string{} - if len(sftpIDFlagValues.IdentityFilePath) > 0 { + identityFiles, err := copy.scanSSHIdentityFiles() + if err != nil { + return xerrors.Errorf("failed to find SSH identity files: %w", err) + } + + err = copy.copySftpId(identityFiles) + if err != nil { + return xerrors.Errorf("failed to perform copy-sftp-id: %w", err) + } + + return nil +} + +func (copy *CopySftpIdCommand) scanSSHIdentityFiles() ([]string, error) { + if len(copy.sftpIDFlagValues.IdentityFilePath) > 0 { // if identity file is given via flag - identityFilePath := commons.MakeLocalPath(sftpIDFlagValues.IdentityFilePath) - identityFiles = append(identityFiles, identityFilePath) - } else { - // scan defaults - identityFiles, err = scanSSHIdentityFiles() + identityFilePath := commons.MakeLocalPath(copy.sftpIDFlagValues.IdentityFilePath) + _, err := os.Stat(identityFilePath) if err != nil { - return xerrors.Errorf("failed to scan ssh identity files: %w", err) + return nil, err } - } - if len(identityFiles) == 0 { - return xerrors.Errorf("failed to find SSH identity files") + return []string{identityFilePath}, nil } - err = copySftpId(filesystem, forceFlagValues, dryRunFlagValues, identityFiles) + // scan defaults + identityFiles, err := copy.scanDefaultSSHIdentityFiles() if err != nil { - return xerrors.Errorf("failed to perform copy-sftp-id: %w", err) + return nil, err } - return nil + + if len(identityFiles) == 0 { + return nil, xerrors.Errorf("failed to find SSH identity files") + } + + return identityFiles, nil } -func scanSSHIdentityFiles() ([]string, error) { +func (copy *CopySftpIdCommand) scanDefaultSSHIdentityFiles() ([]string, error) { // ~/.ssh/*.pub homePath, err := os.UserHomeDir() if err != nil { @@ -120,23 +163,23 @@ func scanSSHIdentityFiles() ([]string, error) { return identityFiles, nil } -func copySftpId(filesystem *irodsclient_fs.FileSystem, forceFlagValues *flag.ForceFlagValues, dryRunFlagValues *flag.DryRunFlagValues, identityFiles []string) error { +func (copy *CopySftpIdCommand) copySftpId(identityFiles []string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "CopySftpIdCommand", "function": "copySftpId", }) - account := commons.GetAccount() - home := commons.GetHomeDir() irodsSshPath := path.Join(home, ".ssh") authorizedKeyPath := path.Join(irodsSshPath, "authorized_keys") - if !filesystem.ExistsDir(irodsSshPath) { - logger.Debugf("SSH directory %s does not exist on iRODS for user %s, creating one", irodsSshPath, account.ClientUser) - if !dryRunFlagValues.DryRun { + if !copy.filesystem.ExistsDir(irodsSshPath) { + logger.Debugf("SSH directory %s does not exist on iRODS for user %s, creating one", irodsSshPath, copy.account.ClientUser) + + if !copy.dryRunFlagValues.DryRun { // create ssh dir - err := filesystem.MakeDir(irodsSshPath, true) + err := copy.filesystem.MakeDir(irodsSshPath, true) if err != nil { return xerrors.Errorf("failed to make dir %s: %w", irodsSshPath, err) } @@ -145,10 +188,10 @@ func copySftpId(filesystem *irodsclient_fs.FileSystem, forceFlagValues *flag.For // read existing authorized_keys authorizedKeysArray := []string{} - if filesystem.ExistsFile(authorizedKeyPath) { - logger.Debugf("reading authorized_keys %s on iRODS for user %s", authorizedKeyPath, account.ClientUser) + if copy.filesystem.ExistsFile(authorizedKeyPath) { + logger.Debugf("reading authorized_keys %s on iRODS for user %s", authorizedKeyPath, copy.account.ClientUser) - handle, err := filesystem.OpenFile(authorizedKeyPath, "", "r") + handle, err := copy.filesystem.OpenFile(authorizedKeyPath, "", "r") if err != nil { return xerrors.Errorf("failed to open file %s: %w", authorizedKeyPath, err) } @@ -181,7 +224,7 @@ func copySftpId(filesystem *irodsclient_fs.FileSystem, forceFlagValues *flag.For contentChanged := false // add for _, identityFile := range identityFiles { - logger.Debugf("copying a SSH public key %s to iRODS for user %s", identityFile, account.ClientUser) + logger.Debugf("copying a SSH public key %s to iRODS for user %s", identityFile, copy.account.ClientUser) // copy // read the identity file first @@ -192,10 +235,10 @@ func copySftpId(filesystem *irodsclient_fs.FileSystem, forceFlagValues *flag.For userKey, _, _, _, err := ssh.ParseAuthorizedKey(identityFileContent) if err != nil { - return xerrors.Errorf("failed to parse a SSH public key %s for user %s: %w", identityFile, account.ClientUser, err) + return xerrors.Errorf("failed to parse a SSH public key %s for user %s: %w", identityFile, copy.account.ClientUser, err) } - if forceFlagValues.Force { + if copy.forceFlagValues.Force { // append forcefully authorizedKeysArray = append(authorizedKeysArray, string(identityFileContent)) contentChanged = true @@ -234,14 +277,14 @@ func copySftpId(filesystem *irodsclient_fs.FileSystem, forceFlagValues *flag.For } // upload - if !dryRunFlagValues.DryRun { + if !copy.dryRunFlagValues.DryRun { if !contentChanged { - logger.Debugf("skipping writing authorized_keys %s on iRODS for user %s, nothing changed", authorizedKeyPath, account.ClientUser) + logger.Debugf("skipping writing authorized_keys %s on iRODS for user %s, nothing changed", authorizedKeyPath, copy.account.ClientUser) } else { - logger.Debugf("writing authorized_keys %s on iRODS for user %s", authorizedKeyPath, account.ClientUser) + logger.Debugf("writing authorized_keys %s on iRODS for user %s", authorizedKeyPath, copy.account.ClientUser) // open the file with write truncate mode - handle, err := filesystem.OpenFile(authorizedKeyPath, "", "w+") + handle, err := copy.filesystem.OpenFile(authorizedKeyPath, "", "w+") if err != nil { return xerrors.Errorf("failed to open file %s: %w", authorizedKeyPath, err) } From 04090808cb1abbc43d869e461a73617244670c56 Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Tue, 13 Aug 2024 11:23:18 -0700 Subject: [PATCH 04/16] refactor code --- cmd/flag/common.go | 8 +- cmd/flag/dry_run.go | 2 +- cmd/gocmd.go | 40 +- cmd/subcmd/addmeta.go | 12 +- cmd/subcmd/bclean.go | 8 +- cmd/subcmd/bput.go | 487 ++++++++++++++------ cmd/subcmd/bun.go | 22 +- cmd/subcmd/cat.go | 10 +- cmd/subcmd/cd.go | 8 +- cmd/subcmd/copy-sftp-id.go | 212 +++++---- cmd/subcmd/cp.go | 576 +++++++++++++++-------- cmd/subcmd/env.go | 112 ++++- cmd/subcmd/get.go | 845 +++++++++++++++++++--------------- cmd/subcmd/ls.go | 60 +-- cmd/subcmd/lsmeta.go | 204 +++++---- cmd/subcmd/lsticket.go | 239 +++++----- cmd/subcmd/mkdir.go | 68 ++- cmd/subcmd/mkticket.go | 61 ++- cmd/subcmd/modticket.go | 148 +++--- cmd/subcmd/mv.go | 189 ++++++-- cmd/subcmd/passwd.go | 54 ++- cmd/subcmd/ps.go | 59 ++- cmd/subcmd/put.go | 917 ++++++++++++++++++++++++------------- cmd/subcmd/pwd.go | 37 +- cmd/subcmd/rm.go | 94 ++-- cmd/subcmd/rmdir.go | 84 +++- cmd/subcmd/rmmeta.go | 238 ++++++---- cmd/subcmd/rmticket.go | 52 ++- cmd/subcmd/svrinfo.go | 51 ++- cmd/subcmd/sync.go | 239 +++++----- cmd/subcmd/upgrade.go | 49 +- commons/bundle_transfer.go | 208 ++++----- commons/commands.go | 103 +---- commons/datetime.go | 2 +- commons/encrypt_pgp.go | 12 +- commons/encrypt_ssh.go | 10 +- commons/encrypt_ssh_rsa.go | 10 +- commons/encrypt_winscp.go | 10 +- commons/errors.go | 68 +++ commons/parallel.go | 2 +- commons/path.go | 36 +- commons/staging.go | 40 +- commons/tar.go | 35 +- commons/transfer_report.go | 87 +++- commons/unit.go | 8 +- 45 files changed, 3634 insertions(+), 2182 deletions(-) create mode 100644 commons/errors.go diff --git a/cmd/flag/common.go b/cmd/flag/common.go index a2722e9..1cba81c 100644 --- a/cmd/flag/common.go +++ b/cmd/flag/common.go @@ -33,7 +33,7 @@ var ( ) func SetCommonFlags(command *cobra.Command, noResource bool) { - command.Flags().StringVarP(&commonFlagValues.ConfigFilePath, "config", "c", "", fmt.Sprintf("Set config file or dir (default \"%s\")", commons.GetDefaultIRODSConfigPath())) + command.Flags().StringVarP(&commonFlagValues.ConfigFilePath, "config", "c", "", fmt.Sprintf("Set config file or directory (default %q)", commons.GetDefaultIRODSConfigPath())) command.Flags().BoolVarP(&commonFlagValues.ShowVersion, "version", "v", false, "Print version") command.Flags().BoolVarP(&commonFlagValues.ShowHelp, "help", "h", false, "Print help") command.Flags().BoolVarP(&commonFlagValues.DebugMode, "debug", "d", false, "Enable debug mode") @@ -116,7 +116,7 @@ func ProcessCommonFlags(command *cobra.Command) (bool, error) { // user defined config file err := commons.LoadConfigFromFile(myCommonFlagValues.ConfigFilePath) if err != nil { - return false, xerrors.Errorf("failed to load config from file %s: %w", myCommonFlagValues.ConfigFilePath, err) // stop here + return false, xerrors.Errorf("failed to load config from file %q: %w", myCommonFlagValues.ConfigFilePath, err) // stop here } readConfig = true @@ -127,7 +127,7 @@ func ProcessCommonFlags(command *cobra.Command) (bool, error) { if len(irodsEnvironmentFileEnvVal) > 0 { err := commons.LoadConfigFromFile(irodsEnvironmentFileEnvVal) if err != nil { - return false, xerrors.Errorf("failed to load config file %s: %w", irodsEnvironmentFileEnvVal, err) // stop here + return false, xerrors.Errorf("failed to load config file %q: %w", irodsEnvironmentFileEnvVal, err) // stop here } readConfig = true @@ -176,7 +176,7 @@ func ProcessCommonFlags(command *cobra.Command) (bool, error) { syncAccount := false if myCommonFlagValues.ResourceUpdated { appConfig.DefaultResource = myCommonFlagValues.Resource - logger.Debugf("use default resource server - %s", appConfig.DefaultResource) + logger.Debugf("use default resource server %q", appConfig.DefaultResource) syncAccount = true } diff --git a/cmd/flag/dry_run.go b/cmd/flag/dry_run.go index 4578d3f..a125a44 100644 --- a/cmd/flag/dry_run.go +++ b/cmd/flag/dry_run.go @@ -13,7 +13,7 @@ var ( ) func SetDryRunFlags(command *cobra.Command) { - command.Flags().BoolVar(&dryRunFlagValues.DryRun, "dry_run", false, "Do not actually perform changes") + command.Flags().BoolVar(&dryRunFlagValues.DryRun, "dry_run", false, "Do not actually change") } func GetDryRunFlagValues() *DryRunFlagValues { diff --git a/cmd/gocmd.go b/cmd/gocmd.go index e2b58df..e4e3e30 100644 --- a/cmd/gocmd.go +++ b/cmd/gocmd.go @@ -8,6 +8,7 @@ import ( irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/cmd/subcmd" + "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -105,11 +106,11 @@ func main() { logger.Errorf("%+v", err) if os.IsNotExist(err) { - fmt.Fprintf(os.Stderr, "File or dir not found!\n") + fmt.Fprintf(os.Stderr, "File or directory not found!\n") } else if irodsclient_types.IsConnectionConfigError(err) { var connectionConfigError *irodsclient_types.ConnectionConfigError if errors.As(err, &connectionConfigError) { - fmt.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server (host: '%s', port: '%d')!\n", connectionConfigError.Config.Host, connectionConfigError.Config.Port) + fmt.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server (host: %q, port: %d)!\n", connectionConfigError.Config.Host, connectionConfigError.Config.Port) } else { fmt.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server!\n") } @@ -125,55 +126,64 @@ func main() { } else if irodsclient_types.IsAuthError(err) { var authError *irodsclient_types.AuthError if errors.As(err, &authError) { - fmt.Fprintf(os.Stderr, "Authentication failed (auth scheme: '%s', username: '%s', zone: '%s')!\n", authError.Config.AuthenticationScheme, authError.Config.ClientUser, authError.Config.ClientZone) + fmt.Fprintf(os.Stderr, "Authentication failed (auth scheme: %q, username: %q, zone: %q)!\n", authError.Config.AuthenticationScheme, authError.Config.ClientUser, authError.Config.ClientZone) } else { fmt.Fprintf(os.Stderr, "Authentication failed!\n") } } else if irodsclient_types.IsFileNotFoundError(err) { var fileNotFoundError *irodsclient_types.FileNotFoundError if errors.As(err, &fileNotFoundError) { - fmt.Fprintf(os.Stderr, "File or dir '%s' not found!\n", fileNotFoundError.Path) + fmt.Fprintf(os.Stderr, "File or directory %q is not found!\n", fileNotFoundError.Path) } else { - fmt.Fprintf(os.Stderr, "File or dir not found!\n") + fmt.Fprintf(os.Stderr, "File or directory is not found!\n") } } else if irodsclient_types.IsCollectionNotEmptyError(err) { var collectionNotEmptyError *irodsclient_types.CollectionNotEmptyError if errors.As(err, &collectionNotEmptyError) { - fmt.Fprintf(os.Stderr, "Dir '%s' not empty!\n", collectionNotEmptyError.Path) + fmt.Fprintf(os.Stderr, "Directory %q is not empty!\n", collectionNotEmptyError.Path) } else { - fmt.Fprintf(os.Stderr, "Dir not empty!\n") + fmt.Fprintf(os.Stderr, "Directory is not empty!\n") } } else if irodsclient_types.IsFileAlreadyExistError(err) { var fileAlreadyExistError *irodsclient_types.FileAlreadyExistError if errors.As(err, &fileAlreadyExistError) { - fmt.Fprintf(os.Stderr, "File or dir '%s' already exist!\n", fileAlreadyExistError.Path) + fmt.Fprintf(os.Stderr, "File or directory %q already exists!\n", fileAlreadyExistError.Path) } else { - fmt.Fprintf(os.Stderr, "File or dir already exist!\n") + fmt.Fprintf(os.Stderr, "File or directory already exists!\n") } } else if irodsclient_types.IsTicketNotFoundError(err) { var ticketNotFoundError *irodsclient_types.TicketNotFoundError if errors.As(err, &ticketNotFoundError) { - fmt.Fprintf(os.Stderr, "Ticket '%s' not found!\n", ticketNotFoundError.Ticket) + fmt.Fprintf(os.Stderr, "Ticket %q is not found!\n", ticketNotFoundError.Ticket) } else { - fmt.Fprintf(os.Stderr, "Ticket not found!\n") + fmt.Fprintf(os.Stderr, "Ticket is not found!\n") } } else if irodsclient_types.IsUserNotFoundError(err) { var userNotFoundError *irodsclient_types.UserNotFoundError if errors.As(err, &userNotFoundError) { - fmt.Fprintf(os.Stderr, "User '%s' not found!\n", userNotFoundError.Name) + fmt.Fprintf(os.Stderr, "User %q is not found!\n", userNotFoundError.Name) } else { - fmt.Fprintf(os.Stderr, "User not found!\n") + fmt.Fprintf(os.Stderr, "User is not found!\n") } } else if irodsclient_types.IsIRODSError(err) { var irodsError *irodsclient_types.IRODSError if errors.As(err, &irodsError) { - fmt.Fprintf(os.Stderr, "iRODS Error (code: '%d', message: '%s')\n", irodsError.Code, irodsError.Error()) + fmt.Fprintf(os.Stderr, "iRODS Error (code: '%d', message: %q)\n", irodsError.Code, irodsError.Error()) } else { fmt.Fprintf(os.Stderr, "iRODS Error!\n") } + } else if commons.IsNotDirError(err) { + var notDirError *commons.NotDirError + if errors.As(err, ¬DirError) { + fmt.Fprintf(os.Stderr, "Destination %q is not a director!\n", notDirError.Path) + } else { + fmt.Fprintf(os.Stderr, "Destination is not a director!\n") + } + } else { + fmt.Fprintf(os.Stderr, "Unexpected error!\nError Trace:\n - %+v\n", err) } - fmt.Fprintf(os.Stderr, "\nError Trace:\n - %+v\n", err) + //fmt.Fprintf(os.Stderr, "\nError Trace:\n - %+v\n", err) os.Exit(1) } } diff --git a/cmd/subcmd/addmeta.go b/cmd/subcmd/addmeta.go index 5b4f6af..c9729be 100644 --- a/cmd/subcmd/addmeta.go +++ b/cmd/subcmd/addmeta.go @@ -128,11 +128,11 @@ func (addMeta *AddMetaCommand) addMetaToPath(targetPath string, attribute string zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - logger.Debugf("add metadata to path %s (attr %s, value %s, unit %s)", targetPath, attribute, value, unit) + logger.Debugf("add metadata to path %q (attr %q, value %q, unit %q)", targetPath, attribute, value, unit) err := addMeta.filesystem.AddMetadata(targetPath, attribute, value, unit) if err != nil { - return xerrors.Errorf("failed to add metadata to path %s (attr %s, value %s, unit %s): %w", targetPath, attribute, value, unit, err) + return xerrors.Errorf("failed to add metadata to path %q (attr %q, value %q, unit %q): %w", targetPath, attribute, value, unit, err) } return nil @@ -145,11 +145,11 @@ func (addMeta *AddMetaCommand) addMetaToUser(username string, attribute string, "function": "addMetaToUser", }) - logger.Debugf("add metadata to user %s (attr %s, value %s, unit %s)", username, attribute, value, unit) + logger.Debugf("add metadata to user %q (attr %q, value %q, unit %q)", username, attribute, value, unit) err := addMeta.filesystem.AddUserMetadata(username, attribute, value, unit) if err != nil { - return xerrors.Errorf("failed to add metadata to user %s (attr %s, value %s, unit %s): %w", username, attribute, value, unit, err) + return xerrors.Errorf("failed to add metadata to user %q (attr %q, value %q, unit %q): %w", username, attribute, value, unit, err) } return nil @@ -162,11 +162,11 @@ func (addMeta *AddMetaCommand) addMetaToResource(resource string, attribute stri "function": "addMetaToResource", }) - logger.Debugf("add metadata to resource %s (attr %s, value %s, unit %s)", resource, attribute, value, unit) + logger.Debugf("add metadata to resource %q (attr %q, value %q, unit %q)", resource, attribute, value, unit) err := addMeta.filesystem.AddUserMetadata(resource, attribute, value, unit) if err != nil { - return xerrors.Errorf("failed to add metadata to resource %s (attr %s, value %s, unit %s): %w", resource, attribute, value, unit, err) + return xerrors.Errorf("failed to add metadata to resource %q (attr %q, value %q, unit %q): %w", resource, attribute, value, unit, err) } return nil diff --git a/cmd/subcmd/bclean.go b/cmd/subcmd/bclean.go index ee94c27..daf7eb8 100644 --- a/cmd/subcmd/bclean.go +++ b/cmd/subcmd/bclean.go @@ -59,7 +59,7 @@ func NewBcleanCommand(command *cobra.Command, args []string) (*BcleanCommand, er } // path - bclean.targetPaths = args[:] + bclean.targetPaths = args return bclean, nil } @@ -100,7 +100,7 @@ func (bclean *BcleanCommand) Process() error { // clear remote if len(bclean.bundleTempFlagValues.IRODSTempPath) > 0 { - logger.Debugf("clearing irods temp dir %s", bclean.bundleTempFlagValues.IRODSTempPath) + logger.Debugf("clearing an irods temp directory %q", bclean.bundleTempFlagValues.IRODSTempPath) commons.CleanUpOldIRODSBundles(bclean.filesystem, bclean.bundleTempFlagValues.IRODSTempPath, true, bclean.forceFlagValues.Force) } else { @@ -130,13 +130,13 @@ func (bclean *BcleanCommand) cleanOne(targetPath string) { if commons.IsStagingDirInTargetPath(targetPath) { // target is staging dir - logger.Debugf("clearing irods target dir %s", targetPath) + logger.Debugf("clearing an irods target directory %q", targetPath) commons.CleanUpOldIRODSBundles(bclean.filesystem, targetPath, true, bclean.forceFlagValues.Force) return } stagingDirPath := commons.GetDefaultStagingDirInTargetPath(targetPath) - logger.Debugf("clearing irods target dir %s", stagingDirPath) + logger.Debugf("clearing an irods target directory %q", stagingDirPath) commons.CleanUpOldIRODSBundles(bclean.filesystem, stagingDirPath, true, bclean.forceFlagValues.Force) } diff --git a/cmd/subcmd/bput.go b/cmd/subcmd/bput.go index d375ad0..5660e27 100644 --- a/cmd/subcmd/bput.go +++ b/cmd/subcmd/bput.go @@ -1,8 +1,11 @@ package subcmd import ( + "io/fs" "os" + "path" "path/filepath" + "time" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" @@ -36,13 +39,24 @@ func AddBputCommand(rootCmd *cobra.Command) { flag.SetDifferentialTransferFlags(bputCmd, true) flag.SetNoRootFlags(bputCmd) flag.SetSyncFlags(bputCmd) + flag.SetTransferReportFlags(putCmd) rootCmd.AddCommand(bputCmd) } +func processBputCommand(command *cobra.Command, args []string) error { + bput, err := NewBputCommand(command, args) + if err != nil { + return err + } + + return bput.Process() +} + type BputCommand struct { command *cobra.Command + forceFlagValues *flag.ForceFlagValues bundleTempFlagValues *flag.BundleTempFlagValues bundleClearFlagValues *flag.BundleClearFlagValues bundleConfigFlagValues *flag.BundleConfigFlagValues @@ -50,8 +64,11 @@ type BputCommand struct { progressFlagValues *flag.ProgressFlagValues retryFlagValues *flag.RetryFlagValues differentialTransferFlagValues *flag.DifferentialTransferFlagValues + checksumFlagValues *flag.ChecksumFlagValues noRootFlagValues *flag.NoRootFlagValues syncFlagValues *flag.SyncFlagValues + postTransferFlagValues *flag.PostTransferFlagValues + transferReportFlagValues *flag.TransferReportFlagValues maxConnectionNum int @@ -60,12 +77,17 @@ type BputCommand struct { sourcePaths []string targetPath string + + bundleTransferManager *commons.BundleTransferManager + transferReportManager *commons.TransferReportManager + updatedPathMap map[string]bool } func NewBputCommand(command *cobra.Command, args []string) (*BputCommand, error) { bput := &BputCommand{ command: command, + forceFlagValues: flag.GetForceFlagValues(), bundleTempFlagValues: flag.GetBundleTempFlagValues(), bundleClearFlagValues: flag.GetBundleClearFlagValues(), bundleConfigFlagValues: flag.GetBundleConfigFlagValues(), @@ -73,31 +95,41 @@ func NewBputCommand(command *cobra.Command, args []string) (*BputCommand, error) progressFlagValues: flag.GetProgressFlagValues(), retryFlagValues: flag.GetRetryFlagValues(), differentialTransferFlagValues: flag.GetDifferentialTransferFlagValues(), + checksumFlagValues: flag.GetChecksumFlagValues(), noRootFlagValues: flag.GetNoRootFlagValues(), syncFlagValues: flag.GetSyncFlagValues(), + postTransferFlagValues: flag.GetPostTransferFlagValues(), + transferReportFlagValues: flag.GetTransferReportFlagValues(command), + + updatedPathMap: map[string]bool{}, } bput.maxConnectionNum = bput.parallelTransferFlagValues.ThreadNumber + 2 + 2 // 2 for metadata op, 2 for extraction // path bput.targetPath = "./" - bput.sourcePaths = args[:] + bput.sourcePaths = args if len(args) >= 2 { bput.targetPath = args[len(args)-1] bput.sourcePaths = args[:len(args)-1] } + if bput.noRootFlagValues.NoRoot && len(bput.sourcePaths) > 1 { + return nil, xerrors.Errorf("failed to put multiple source collections without creating root directory") + } + return bput, nil } -func processBputCommand(command *cobra.Command, args []string) error { +func (bput *BputCommand) Process() error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "processBputCommand", + "struct": "BputCommand", + "function": "Process", }) - cont, err := flag.ProcessCommonFlags(command) + cont, err := flag.ProcessCommonFlags(bput.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -112,132 +144,105 @@ func processBputCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - bundleTempFlagValues := flag.GetBundleTempFlagValues() - bundleClearFlagValues := flag.GetBundleClearFlagValues() - bundleConfigFlagValues := flag.GetBundleConfigFlagValues() - parallelTransferFlagValues := flag.GetParallelTransferFlagValues() - progressFlagValues := flag.GetProgressFlagValues() - retryFlagValues := flag.GetRetryFlagValues() - differentialTransferFlagValues := flag.GetDifferentialTransferFlagValues() - noRootFlagValues := flag.GetNoRootFlagValues() - syncFlagValues := flag.GetSyncFlagValues() - - maxConnectionNum := parallelTransferFlagValues.ThreadNumber + 2 + 2 // 2 for metadata op, 2 for extraction - // clear local - if bundleClearFlagValues.Clear { - commons.CleanUpOldLocalBundles(bundleTempFlagValues.LocalTempPath, true) + // delete local bundles before entering to retry + if bput.bundleClearFlagValues.Clear { + commons.CleanUpOldLocalBundles(bput.bundleTempFlagValues.LocalTempPath, true) } - if retryFlagValues.RetryNumber > 0 && !retryFlagValues.RetryChild { - err = commons.RunWithRetry(retryFlagValues.RetryNumber, retryFlagValues.RetryIntervalSeconds) + // handle retry + if bput.retryFlagValues.RetryNumber > 0 && !bput.retryFlagValues.RetryChild { + err = commons.RunWithRetry(bput.retryFlagValues.RetryNumber, bput.retryFlagValues.RetryIntervalSeconds) if err != nil { - return xerrors.Errorf("failed to run with retry %d: %w", retryFlagValues.RetryNumber, err) + return xerrors.Errorf("failed to run with retry %d: %w", bput.retryFlagValues.RetryNumber, err) } return nil } // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClientAdvanced(account, maxConnectionNum, parallelTransferFlagValues.TCPBufferSize) + bput.account = commons.GetAccount() + bput.filesystem, err = commons.GetIRODSFSClientAdvanced(bput.account, bput.maxConnectionNum, bput.parallelTransferFlagValues.TCPBufferSize) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer bput.filesystem.Release() - defer filesystem.Release() - - targetPath := "./" - sourcePaths := args[:] - - if len(args) >= 2 { - targetPath = args[len(args)-1] - sourcePaths = args[:len(args)-1] + // transfer report + bput.transferReportManager, err = commons.NewTransferReportManager(bput.transferReportFlagValues.Report, bput.transferReportFlagValues.ReportPath, bput.transferReportFlagValues.ReportToStdout) + if err != nil { + return xerrors.Errorf("failed to create transfer report manager: %w", err) } + defer bput.transferReportManager.Release() - if noRootFlagValues.NoRoot && len(sourcePaths) > 1 { - return xerrors.Errorf("failed to bput multiple source dirs without creating root directory") + // run + // target must be a dir + err = bput.ensureTargetIsDir(bput.targetPath) + if err != nil { + return err } - cwd := commons.GetCWD() - home := commons.GetHomeDir() - zone := commons.GetZone() - targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - - _, err = filesystem.StatDir(targetPath) + // get staging path + stagingDirPath, err := bput.getStagingDir(bput.targetPath) if err != nil { - return xerrors.Errorf("failed to stat dir %s: %w", targetPath, err) + return err } - logger.Info("determining staging dir...") - if len(bundleTempFlagValues.IRODSTempPath) > 0 { - logger.Debugf("validating staging dir - %s", bundleTempFlagValues.IRODSTempPath) - - bundleTempFlagValues.IRODSTempPath = commons.MakeIRODSPath(cwd, home, zone, bundleTempFlagValues.IRODSTempPath) - ok, err := commons.ValidateStagingDir(filesystem, targetPath, bundleTempFlagValues.IRODSTempPath) + // clear old irods bundles + if bput.bundleClearFlagValues.Clear { + logger.Debugf("clearing an irods temp directory %q", stagingDirPath) + err = commons.CleanUpOldIRODSBundles(bput.filesystem, stagingDirPath, false, true) if err != nil { - return xerrors.Errorf("failed to validate staging dir - %s: %w", bundleTempFlagValues.IRODSTempPath, err) - } - - if !ok { - logger.Debugf("unable to use the given staging dir %s since it is in a different resource server, using default staging dir", bundleTempFlagValues.IRODSTempPath) - return xerrors.Errorf("staging dir %s is in a different resource server", bundleTempFlagValues.IRODSTempPath) + return xerrors.Errorf("failed to clean up old irods bundle files in %q: %w", stagingDirPath, err) } - } else { - // set default staging dir - logger.Debug("get default staging dir") - - bundleTempFlagValues.IRODSTempPath = commons.GetDefaultStagingDir(targetPath) } - err = commons.CheckSafeStagingDir(bundleTempFlagValues.IRODSTempPath) + // bundle root path + bundleRootPath := "/" + bundleRootPath, err = commons.GetCommonRootLocalDirPath(bput.sourcePaths) if err != nil { - return xerrors.Errorf("failed to get safe staging dir: %w", err) + return xerrors.Errorf("failed to get a common root directory for source paths: %w", err) } - logger.Infof("use staging dir - %s", bundleTempFlagValues.IRODSTempPath) - - if bundleClearFlagValues.Clear { - logger.Debugf("clearing irods temp dir %s", bundleTempFlagValues.IRODSTempPath) - commons.CleanUpOldIRODSBundles(filesystem, bundleTempFlagValues.IRODSTempPath, false, true) + if !bput.noRootFlagValues.NoRoot { + // use parent dir + bundleRootPath = filepath.Dir(bundleRootPath) } - bundleTransferManager := commons.NewBundleTransferManager(filesystem, targetPath, bundleConfigFlagValues.MaxFileNum, bundleConfigFlagValues.MaxFileSize, parallelTransferFlagValues.SingleTread, parallelTransferFlagValues.ThreadNumber, parallelTransferFlagValues.RedirectToResource, parallelTransferFlagValues.Icat, bundleTempFlagValues.LocalTempPath, bundleTempFlagValues.IRODSTempPath, differentialTransferFlagValues.DifferentialTransfer, differentialTransferFlagValues.NoHash, bundleConfigFlagValues.NoBulkRegistration, progressFlagValues.ShowProgress, progressFlagValues.ShowFullPath) - bundleTransferManager.Start() + // bundle transfer manager + bput.bundleTransferManager = commons.NewBundleTransferManager(bput.filesystem, bput.targetPath, bundleRootPath, bput.bundleConfigFlagValues.MaxFileNum, bput.bundleConfigFlagValues.MaxFileSize, bput.parallelTransferFlagValues.SingleTread, bput.parallelTransferFlagValues.ThreadNumber, bput.parallelTransferFlagValues.RedirectToResource, bput.parallelTransferFlagValues.Icat, bput.bundleTempFlagValues.LocalTempPath, bput.bundleTempFlagValues.IRODSTempPath, bput.differentialTransferFlagValues.DifferentialTransfer, bput.differentialTransferFlagValues.NoHash, bput.bundleConfigFlagValues.NoBulkRegistration, bput.progressFlagValues.ShowProgress, bput.progressFlagValues.ShowFullPath) + bput.bundleTransferManager.Start() - if noRootFlagValues.NoRoot && len(sourcePaths) == 1 { - bundleRootPath, err := commons.GetCommonRootLocalDirPathForSync(sourcePaths) + // run + for _, sourcePath := range bput.sourcePaths { + err = bput.bputOne(sourcePath) if err != nil { - return xerrors.Errorf("failed to get common root dir for source paths: %w", err) + return xerrors.Errorf("failed to bundle-put %q to %q: %w", sourcePath, bput.targetPath, err) } - - bundleTransferManager.SetBundleRootPath(bundleRootPath) - } else { - bundleRootPath, err := commons.GetCommonRootLocalDirPath(sourcePaths) - if err != nil { - return xerrors.Errorf("failed to get common root dir for source paths: %w", err) - } - - bundleTransferManager.SetBundleRootPath(bundleRootPath) } - for _, sourcePath := range sourcePaths { - err = bputOne(bundleTransferManager, sourcePath) - if err != nil { - return xerrors.Errorf("failed to perform bput %s to %s: %w", sourcePath, targetPath, err) - } + bput.bundleTransferManager.DoneScheduling() + err = bput.bundleTransferManager.Wait() + if err != nil { + return xerrors.Errorf("failed to bundle-put: %w", err) } - bundleTransferManager.DoneScheduling() - err = bundleTransferManager.Wait() - if err != nil { - return xerrors.Errorf("failed to perform bundle transfer: %w", err) + // delete on success + if bput.postTransferFlagValues.DeleteOnSuccess { + for _, sourcePath := range bput.sourcePaths { + logger.Infof("deleting source %q after successful data put", sourcePath) + + err := bput.deleteOnSuccess(sourcePath) + if err != nil { + return xerrors.Errorf("failed to delete source %q: %w", sourcePath, err) + } + } } // delete extra - if syncFlagValues.Delete { - logger.Infof("deleting extra files and dirs under %s", targetPath) + if bput.syncFlagValues.Delete { + logger.Infof("deleting extra files and directories under %q", bput.targetPath) - err = bputDeleteExtra(bundleTransferManager, targetPath) + err = bput.deleteExtra(bput.targetPath) if err != nil { return xerrors.Errorf("failed to delete extra files: %w", err) } @@ -246,107 +251,299 @@ func processBputCommand(command *cobra.Command, args []string) error { return nil } -func bputOne(bundleManager *commons.BundleTransferManager, sourcePath string) error { +func (bput *BputCommand) ensureTargetIsDir(targetPath string) error { + cwd := commons.GetCWD() + home := commons.GetHomeDir() + zone := commons.GetZone() + targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) + + targetEntry, err := bput.filesystem.Stat(targetPath) + if err != nil { + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + + if !targetEntry.IsDir() { + return commons.NewNotDirError(targetPath) + } + + return nil +} + +func (bput *BputCommand) getStagingDir(targetPath string) (string, error) { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "bputOne", + "struct": "BputCommand", + "function": "getStagingDir", }) - sourcePath = commons.MakeLocalPath(sourcePath) + cwd := commons.GetCWD() + home := commons.GetHomeDir() + zone := commons.GetZone() + targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) + + if len(bput.bundleTempFlagValues.IRODSTempPath) > 0 { + stagingPath := commons.MakeIRODSPath(cwd, home, zone, bput.bundleTempFlagValues.IRODSTempPath) + + createdDir := false + tempEntry, err := bput.filesystem.Stat(stagingPath) + if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // not exist + err = bput.filesystem.MakeDir(stagingPath, true) + if err != nil { + // failed to + return "", xerrors.Errorf("failed to make a collection %q: %w", stagingPath, err) + } + createdDir = true + } else { + return "", xerrors.Errorf("failed to stat %q: %w", stagingPath, err) + } + } + + if !tempEntry.IsDir() { + return "", xerrors.Errorf("staging path %q is a file", stagingPath) + } + + // is it safe? + logger.Debugf("validating staging directory %q", stagingPath) + + err = commons.IsSafeStagingDir(stagingPath) + if err != nil { + logger.Debugf("staging path %q is not safe", stagingPath) + + if createdDir { + bput.filesystem.RemoveDir(stagingPath, true, true) + } + + return "", xerrors.Errorf("staging path %q is not safe: %w", stagingPath, err) + } + + ok, err := commons.IsSameResourceServer(bput.filesystem, targetPath, stagingPath) + if err != nil { + logger.Debugf("failed to validate staging directory %q and target %q - %s", stagingPath, targetPath, err.Error()) + + if createdDir { + bput.filesystem.RemoveDir(stagingPath, true, true) + } + + stagingPath = commons.GetDefaultStagingDir(targetPath) + logger.Debugf("use default staging path %q for target %q - %s", stagingPath, targetPath, err.Error()) + return stagingPath, nil + } + + if !ok { + logger.Debugf("staging directory %q is in a different resource server as target %q", stagingPath, targetPath) + + if createdDir { + bput.filesystem.RemoveDir(stagingPath, true, true) + } - realSourcePath, err := commons.ResolveSymlink(sourcePath) + stagingPath = commons.GetDefaultStagingDir(targetPath) + logger.Debugf("use default staging path %q for target %q", stagingPath, targetPath) + return stagingPath, nil + } + + logger.Debugf("use staging path %q for target %q", stagingPath, targetPath) + return stagingPath, nil + } + + // use default staging dir + stagingPath := commons.GetDefaultStagingDir(targetPath) + + err := commons.IsSafeStagingDir(stagingPath) if err != nil { - return xerrors.Errorf("failed to resolve symlink %s: %w", sourcePath, err) + logger.Debugf("staging path %q is not safe", stagingPath) + + return "", xerrors.Errorf("staging path %q is not safe: %w", stagingPath, err) } - logger.Debugf("path %s ==> %s", sourcePath, realSourcePath) + // may not exist + err = bput.filesystem.MakeDir(stagingPath, true) + if err != nil { + // failed to + return "", xerrors.Errorf("failed to make a collection %q: %w", stagingPath, err) + } - sourceStat, err := os.Stat(realSourcePath) + logger.Debugf("use default staging path %q for target %q", stagingPath, targetPath) + return stagingPath, nil +} + +func (bput *BputCommand) bputOne(sourcePath string) error { + sourcePath = commons.MakeLocalPath(sourcePath) + + sourceStat, err := os.Stat(sourcePath) if err != nil { if os.IsNotExist(err) { - return irodsclient_types.NewFileNotFoundError(realSourcePath) + return irodsclient_types.NewFileNotFoundError(sourcePath) } - return xerrors.Errorf("failed to stat %s: %w", realSourcePath, err) + return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) } - if !sourceStat.IsDir() { - // file - err = bundleManager.Schedule(sourcePath, false, sourceStat.Size(), sourceStat.ModTime().Local()) - if err != nil { - return xerrors.Errorf("failed to schedule %s: %w", sourcePath, err) - } - } else { + if sourceStat.IsDir() { // dir - logger.Debugf("bundle-uploading a local directory %s", sourcePath) + return bput.putDir(sourceStat, sourcePath) + } + + // file + return bput.putFile(sourceStat, sourcePath) +} + +func (bput *BputCommand) putFile(sourceStat fs.FileInfo, sourcePath string) error { + err := bput.bundleTransferManager.Schedule(sourcePath, false, sourceStat.Size(), sourceStat.ModTime().Local()) + if err != nil { + return xerrors.Errorf("failed to schedule a file %q: %w", sourcePath, err) + } + + //commons.MarkPathMap(bput.updatedPathMap, targetPath) + + return nil +} + +func (bput *BputCommand) putDir(sourceStat fs.FileInfo, sourcePath string) error { + err := bput.bundleTransferManager.Schedule(sourcePath, true, 0, sourceStat.ModTime().Local()) + if err != nil { + return xerrors.Errorf("failed to schedule a directory %q: %w", sourcePath, err) + } - entries, err := os.ReadDir(sourcePath) + entries, err := os.ReadDir(sourcePath) + if err != nil { + return xerrors.Errorf("failed to read a directory %q: %w", sourcePath, err) + } + + for _, entry := range entries { + entryPath := filepath.Join(sourcePath, entry.Name()) + + entryStat, err := os.Stat(entryPath) if err != nil { - return xerrors.Errorf("failed to read dir %s: %w", sourcePath, err) + if os.IsNotExist(err) { + return irodsclient_types.NewFileNotFoundError(entryPath) + } + + return xerrors.Errorf("failed to stat %q: %w", entryPath, err) } - for _, entry := range entries { - entryPath := filepath.Join(sourcePath, entry.Name()) - err = bputOne(bundleManager, entryPath) + if entryStat.IsDir() { + // dir + err = bput.putDir(entryStat, entryPath) + if err != nil { + return err + } + + //commons.MarkPathMap(bput.updatedPathMap, newEntryPath) + } else { + // file + err = bput.putFile(entryStat, entryPath) if err != nil { return err } + + //commons.MarkPathMap(bput.updatedPathMap, newEntryPath) } } + //commons.MarkPathMap(put.updatedPathMap, targetPath) + return nil } -func bputDeleteExtra(bundleManager *commons.BundleTransferManager, targetPath string) error { - pathMap := bundleManager.GetInputPathMap() - filesystem := bundleManager.GetFilesystem() +func (bput *BputCommand) deleteOnSuccess(sourcePath string) error { + sourceStat, err := os.Stat(sourcePath) + if err != nil { + return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) + } - return bputDeleteExtraInternal(filesystem, pathMap, targetPath) + if sourceStat.IsDir() { + return os.RemoveAll(sourcePath) + } + + return os.Remove(sourcePath) } -func bputDeleteExtraInternal(filesystem *irodsclient_fs.FileSystem, inputPathMap map[string]bool, targetPath string) error { +func (bput *BputCommand) deleteExtra(targetPath string) error { + cwd := commons.GetCWD() + home := commons.GetHomeDir() + zone := commons.GetZone() + targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) + + return bput.deleteExtraInternal(targetPath) +} + +func (bput *BputCommand) deleteExtraInternal(targetPath string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "bputDeleteExtraInternal", + "struct": "BputCommand", + "function": "deleteExtraInternal", }) - targetEntry, err := filesystem.Stat(targetPath) + targetEntry, err := bput.filesystem.Stat(targetPath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", targetPath, err) + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } - if targetEntry.Type == irodsclient_fs.FileEntry { - if _, ok := inputPathMap[targetPath]; !ok { + if !targetEntry.IsDir() { + // file + if _, ok := bput.updatedPathMap[targetPath]; !ok { // extra file - logger.Debugf("removing an extra data object %s", targetPath) - removeErr := filesystem.RemoveFile(targetPath, true) - if removeErr != nil { - return removeErr + logger.Debugf("removing an extra data object %q", targetPath) + + removeErr := bput.filesystem.RemoveFile(targetPath, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"extra", "put"}, } - } - } else { - // dir - if _, ok := inputPathMap[targetPath]; !ok { - // extra dir - logger.Debugf("removing an extra collection %s", targetPath) - removeErr := filesystem.RemoveDir(targetPath, true, true) + + bput.transferReportManager.AddFile(reportFile) + if removeErr != nil { return removeErr } - } else { - // non extra dir - entries, err := filesystem.List(targetPath) - if err != nil { - return xerrors.Errorf("failed to list dir %s: %w", targetPath, err) - } + } - for idx := range entries { - newTargetPath := entries[idx].Path + return nil + } - err = bputDeleteExtraInternal(filesystem, inputPathMap, newTargetPath) - if err != nil { - return err - } + // target is dir + if _, ok := bput.updatedPathMap[targetPath]; !ok { + // extra dir + logger.Debugf("removing an extra collection %q", targetPath) + + removeErr := bput.filesystem.RemoveDir(targetPath, true, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"extra", "put", "dir"}, + } + + bput.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + // non extra dir + // scan recursively + entries, err := bput.filesystem.List(targetPath) + if err != nil { + return xerrors.Errorf("failed to list a directory %q: %w", targetPath, err) + } + + for _, entry := range entries { + newTargetPath := path.Join(targetPath, entry.Name) + err = bput.deleteExtraInternal(newTargetPath) + if err != nil { + return err } } } diff --git a/cmd/subcmd/bun.go b/cmd/subcmd/bun.go index f7a61aa..30d3a2c 100644 --- a/cmd/subcmd/bun.go +++ b/cmd/subcmd/bun.go @@ -101,7 +101,7 @@ func (bun *BunCommand) Process() error { if bun.bundleFlagValues.Extract { err = bun.extractOne(sourcePath, bun.targetPath) if err != nil { - return xerrors.Errorf("failed to perform bun %s to %s: %w", sourcePath, bun.targetPath, err) + return xerrors.Errorf("failed to extract bundle file %q to %q: %w", sourcePath, bun.targetPath, err) } } } @@ -122,7 +122,7 @@ func (bun *BunCommand) getDataType(irodsPath string, dataType string) (irodsclie case "": // auto default: - return "", xerrors.Errorf("unknown format %s", dataType) + return "", xerrors.Errorf("unknown format %q", dataType) } // auto @@ -156,35 +156,35 @@ func (bun *BunCommand) extractOne(sourcePath string, targetPath string) error { sourceEntry, err := bun.filesystem.Stat(sourcePath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", sourcePath, err) + return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) } targetEntry, err := bun.filesystem.Stat(targetPath) if err != nil { if !irodsclient_types.IsFileNotFoundError(err) { - return xerrors.Errorf("failed to stat %s: %w", targetPath, err) + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } } else { - if targetEntry.Type == irodsclient_fs.FileEntry { - return xerrors.Errorf("%s is not a collection", targetPath) + if !targetEntry.IsDir() { + return commons.NewNotDirError(targetPath) } } - if sourceEntry.Type != irodsclient_fs.FileEntry { - return xerrors.Errorf("source %s must be a data object", sourcePath) + if sourceEntry.IsDir() { + return xerrors.Errorf("source %q must be a data object", sourcePath) } // file - logger.Debugf("extracting a data object %s to %s", sourcePath, targetPath) + logger.Debugf("extracting a data object %q to %q", sourcePath, targetPath) dt, err := bun.getDataType(sourcePath, bun.bundleFlagValues.DataType) if err != nil { - return xerrors.Errorf("failed to get type %s: %w", sourcePath, err) + return xerrors.Errorf("failed to get type %q: %w", sourcePath, err) } err = bun.filesystem.ExtractStructFile(sourcePath, targetPath, "", dt, bun.forceFlagValues.Force, bun.bundleFlagValues.BulkRegistration) if err != nil { - return xerrors.Errorf("failed to extract file %s to %s: %w", sourcePath, targetPath, err) + return xerrors.Errorf("failed to extract file %q to %q: %w", sourcePath, targetPath, err) } return nil diff --git a/cmd/subcmd/cat.go b/cmd/subcmd/cat.go index a3561e4..781a7a9 100644 --- a/cmd/subcmd/cat.go +++ b/cmd/subcmd/cat.go @@ -90,7 +90,7 @@ func (cat *CatCommand) Process() error { appConfig := commons.GetConfig() syncAccount := false if len(cat.ticketAccessFlagValues.Name) > 0 { - logger.Debugf("use ticket: %s", cat.ticketAccessFlagValues.Name) + logger.Debugf("use ticket: %q", cat.ticketAccessFlagValues.Name) appConfig.Ticket = cat.ticketAccessFlagValues.Name syncAccount = true } @@ -114,7 +114,7 @@ func (cat *CatCommand) Process() error { for _, sourcePath := range cat.sourcePaths { err = cat.catOne(sourcePath) if err != nil { - return xerrors.Errorf("failed to perform cat %s: %w", sourcePath, err) + return xerrors.Errorf("failed to display content of %q: %w", sourcePath, err) } } @@ -129,17 +129,17 @@ func (cat *CatCommand) catOne(sourcePath string) error { sourceEntry, err := cat.filesystem.Stat(sourcePath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", sourcePath, err) + return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) } - if sourceEntry.Type != irodsclient_fs.FileEntry { + if sourceEntry.IsDir() { return xerrors.Errorf("cannot show the content of a collection") } // file fh, err := cat.filesystem.OpenFile(sourcePath, "", "r") if err != nil { - return xerrors.Errorf("failed to open file %s: %w", sourcePath, err) + return xerrors.Errorf("failed to open file %q: %w", sourcePath, err) } defer fh.Close() diff --git a/cmd/subcmd/cd.go b/cmd/subcmd/cd.go index 291bcf1..d63f101 100644 --- a/cmd/subcmd/cd.go +++ b/cmd/subcmd/cd.go @@ -89,7 +89,7 @@ func (cd *CdCommand) Process() error { // run err = cd.changeWorkingDir(cd.targetPath) if err != nil { - return xerrors.Errorf("failed to perform cd %s: %w", cd.targetPath, err) + return xerrors.Errorf("failed to change working directory to %q: %w", cd.targetPath, err) } return nil @@ -113,16 +113,16 @@ func (cd *CdCommand) changeWorkingDir(collectionPath string) error { } defer cd.filesystem.ReturnMetadataConnection(connection) - logger.Debugf("changing working dir: %s", collectionPath) + logger.Debugf("changing working directory to %q", collectionPath) _, err = irodsclient_irodsfs.GetCollection(connection, collectionPath) if err != nil { - return xerrors.Errorf("failed to get collection %s: %w", collectionPath, err) + return xerrors.Errorf("failed to get collection %q: %w", collectionPath, err) } err = commons.SetCWD(collectionPath) if err != nil { - return xerrors.Errorf("failed to set current working collection %s: %w", collectionPath, err) + return xerrors.Errorf("failed to set current working collection to %q: %w", collectionPath, err) } return nil diff --git a/cmd/subcmd/copy-sftp-id.go b/cmd/subcmd/copy-sftp-id.go index 4645651..20859d2 100644 --- a/cmd/subcmd/copy-sftp-id.go +++ b/cmd/subcmd/copy-sftp-id.go @@ -2,7 +2,6 @@ package subcmd import ( "bytes" - "io" "os" "path" "path/filepath" @@ -103,7 +102,7 @@ func (copy *CopySftpIdCommand) Process() error { err = copy.copySftpId(identityFiles) if err != nil { - return xerrors.Errorf("failed to perform copy-sftp-id: %w", err) + return xerrors.Errorf("failed to copy sftp-ID: %w", err) } return nil @@ -138,14 +137,14 @@ func (copy *CopySftpIdCommand) scanDefaultSSHIdentityFiles() ([]string, error) { // ~/.ssh/*.pub homePath, err := os.UserHomeDir() if err != nil { - return nil, xerrors.Errorf("failed to get user home dir: %w", err) + return nil, xerrors.Errorf("failed to get user home directory: %w", err) } sshPath := filepath.Join(homePath, ".ssh") sshDirEntries, err := os.ReadDir(sshPath) if err != nil { - return nil, xerrors.Errorf("failed to read dir %s: %w", sshPath, err) + return nil, xerrors.Errorf("failed to read a directory %q: %w", sshPath, err) } identityFiles := []string{} @@ -163,146 +162,157 @@ func (copy *CopySftpIdCommand) scanDefaultSSHIdentityFiles() ([]string, error) { return identityFiles, nil } -func (copy *CopySftpIdCommand) copySftpId(identityFiles []string) error { +func (copy *CopySftpIdCommand) readAuthorizedKeys(authorizedKeyPath string) ([]string, error) { logger := log.WithFields(log.Fields{ "package": "subcmd", "struct": "CopySftpIdCommand", - "function": "copySftpId", + "function": "readAuthorizedKeys", }) - home := commons.GetHomeDir() - irodsSshPath := path.Join(home, ".ssh") - authorizedKeyPath := path.Join(irodsSshPath, "authorized_keys") - - if !copy.filesystem.ExistsDir(irodsSshPath) { - logger.Debugf("SSH directory %s does not exist on iRODS for user %s, creating one", irodsSshPath, copy.account.ClientUser) - - if !copy.dryRunFlagValues.DryRun { - // create ssh dir - err := copy.filesystem.MakeDir(irodsSshPath, true) - if err != nil { - return xerrors.Errorf("failed to make dir %s: %w", irodsSshPath, err) - } - } - } - - // read existing authorized_keys - authorizedKeysArray := []string{} if copy.filesystem.ExistsFile(authorizedKeyPath) { - logger.Debugf("reading authorized_keys %s on iRODS for user %s", authorizedKeyPath, copy.account.ClientUser) - - handle, err := copy.filesystem.OpenFile(authorizedKeyPath, "", "r") - if err != nil { - return xerrors.Errorf("failed to open file %s: %w", authorizedKeyPath, err) - } - defer handle.Close() - - sb := strings.Builder{} - readBuffer := make([]byte, 1024) - for { - readLen, err := handle.Read(readBuffer) - if err != nil && err != io.EOF { - return xerrors.Errorf("failed to read file %s: %w", authorizedKeyPath, err) - } + logger.Debugf("reading authorized_keys %q on iRODS for user %q", authorizedKeyPath, copy.account.ClientUser) - _, err2 := sb.Write(readBuffer[:readLen]) - if err2 != nil { - return xerrors.Errorf("failed to write to buffer: %w", err2) - } + contentBuffer := bytes.Buffer{} - if err == io.EOF { - break - } + _, err := copy.filesystem.DownloadFileToBuffer(authorizedKeyPath, "", contentBuffer, true, nil) + if err != nil { + return nil, xerrors.Errorf("failed to read file %q: %w", authorizedKeyPath, err) } - existingAuthorizedKeysContent := sb.String() + existingAuthorizedKeysContent := contentBuffer.String() if len(existingAuthorizedKeysContent) > 0 { - authorizedKeysArray = strings.Split(existingAuthorizedKeysContent, "\n") + authorizedKeysArray := strings.Split(existingAuthorizedKeysContent, "\n") + return authorizedKeysArray, nil } } + return []string{}, nil +} + +func (copy *CopySftpIdCommand) updateAuthorizedKeys(identityFiles []string, authorizedKeys []string) ([]string, bool, error) { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "CopySftpIdCommand", + "function": "updateAuthorizedKeys", + }) + contentChanged := false + newAuthorizedKeys := []string{} + + newAuthorizedKeys = append(newAuthorizedKeys, authorizedKeys...) + // add for _, identityFile := range identityFiles { - logger.Debugf("copying a SSH public key %s to iRODS for user %s", identityFile, copy.account.ClientUser) + logger.Debugf("copying a SSH public key %q to iRODS for user %q", identityFile, copy.account.ClientUser) // copy // read the identity file first identityFileContent, err := os.ReadFile(identityFile) if err != nil { - return xerrors.Errorf("failed to read file %s: %w", identityFile, err) + return newAuthorizedKeys, contentChanged, xerrors.Errorf("failed to read file %q: %w", identityFile, err) } userKey, _, _, _, err := ssh.ParseAuthorizedKey(identityFileContent) if err != nil { - return xerrors.Errorf("failed to parse a SSH public key %s for user %s: %w", identityFile, copy.account.ClientUser, err) + return newAuthorizedKeys, contentChanged, xerrors.Errorf("failed to parse a SSH public key %q for user %q: %w", identityFile, copy.account.ClientUser, err) } if copy.forceFlagValues.Force { // append forcefully - authorizedKeysArray = append(authorizedKeysArray, string(identityFileContent)) + newAuthorizedKeys = append(newAuthorizedKeys, string(identityFileContent)) contentChanged = true - } else { - // check if exists, add only if it doesn't - hasExisting := false - for keyLineIdx, keyLine := range authorizedKeysArray { - keyLine = strings.TrimSpace(keyLine) - if keyLine == "" || keyLine[0] == '#' { - // skip - continue - } - - authorizedKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(keyLine)) - if err != nil { - // skip - log.Debugf("failed to parse a authorized key line - %s", err.Error()) - continue - } - - if bytes.Equal(authorizedKey.Marshal(), userKey.Marshal()) { - // existing - update - authorizedKeysArray[keyLineIdx] = string(identityFileContent) - hasExisting = true - contentChanged = true - break - } + continue + } + + // check if exists, add only if it doesn't + hasExisting := false + for keyLineIdx, keyLine := range newAuthorizedKeys { + keyLine = strings.TrimSpace(keyLine) + if keyLine == "" || keyLine[0] == '#' { + // skip + continue + } + + authorizedKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(keyLine)) + if err != nil { + // skip + log.Debugf("failed to parse a authorized key line - %s", err.Error()) + continue } - if !hasExisting { - // not found - add - authorizedKeysArray = append(authorizedKeysArray, string(identityFileContent)) + if bytes.Equal(authorizedKey.Marshal(), userKey.Marshal()) { + // existing - update + newAuthorizedKeys[keyLineIdx] = string(identityFileContent) + hasExisting = true contentChanged = true + break } } + + if !hasExisting { + // not found - add + newAuthorizedKeys = append(newAuthorizedKeys, string(identityFileContent)) + contentChanged = true + } + } + + return newAuthorizedKeys, contentChanged, nil +} + +func (copy *CopySftpIdCommand) copySftpId(identityFiles []string) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "CopySftpIdCommand", + "function": "copySftpId", + }) + + home := commons.GetHomeDir() + irodsSshPath := path.Join(home, ".ssh") + authorizedKeyPath := path.Join(irodsSshPath, "authorized_keys") + + if !copy.filesystem.ExistsDir(irodsSshPath) { + logger.Debugf("SSH directory %q does not exist on iRODS for user %q, creating one", irodsSshPath, copy.account.ClientUser) + + if !copy.dryRunFlagValues.DryRun { + // create ssh dir + err := copy.filesystem.MakeDir(irodsSshPath, true) + if err != nil { + return xerrors.Errorf("failed to make a directory %q: %w", irodsSshPath, err) + } + } + } + + // read existing authorized_keys + authorizedKeys, err := copy.readAuthorizedKeys(authorizedKeyPath) + if err != nil { + return xerrors.Errorf("failed to read authorized_keys %q: %w", authorizedKeyPath, err) + } + + authorizedKeysUpdated, contentChanged, err := copy.updateAuthorizedKeys(identityFiles, authorizedKeys) + if err != nil { + return xerrors.Errorf("failed to update authorized_keys: %w", err) + } + + contentBuf := bytes.Buffer{} + for _, key := range authorizedKeysUpdated { + key = strings.TrimSpace(key) + if len(key) > 0 { + contentBuf.WriteString(key + "\n") + } } // upload if !copy.dryRunFlagValues.DryRun { if !contentChanged { - logger.Debugf("skipping writing authorized_keys %s on iRODS for user %s, nothing changed", authorizedKeyPath, copy.account.ClientUser) - } else { - logger.Debugf("writing authorized_keys %s on iRODS for user %s", authorizedKeyPath, copy.account.ClientUser) + logger.Debugf("skipping writing authorized_keys %q on iRODS for user %q, nothing changed", authorizedKeyPath, copy.account.ClientUser) + return nil + } - // open the file with write truncate mode - handle, err := copy.filesystem.OpenFile(authorizedKeyPath, "", "w+") - if err != nil { - return xerrors.Errorf("failed to open file %s: %w", authorizedKeyPath, err) - } - defer handle.Close() - - buf := bytes.Buffer{} - for _, key := range authorizedKeysArray { - key = strings.TrimSpace(key) - if len(key) > 0 { - buf.WriteString(key) - buf.WriteString("\n") - } - } + logger.Debugf("writing authorized_keys %q on iRODS for user %q", authorizedKeyPath, copy.account.ClientUser) - _, err = handle.Write(buf.Bytes()) - if err != nil { - return xerrors.Errorf("failed to write: %w", err) - } + _, err := copy.filesystem.UploadFileFromBuffer(contentBuf, authorizedKeyPath, "", false, true, true, nil) + if err != nil { + return xerrors.Errorf("failed to update keys in %q: %w", authorizedKeyPath, err) } } diff --git a/cmd/subcmd/cp.go b/cmd/subcmd/cp.go index ac52321..87fc213 100644 --- a/cmd/subcmd/cp.go +++ b/cmd/subcmd/cp.go @@ -2,7 +2,10 @@ package subcmd import ( "bytes" + "encoding/hex" "fmt" + "path" + "time" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" @@ -34,17 +37,80 @@ func AddCpCommand(rootCmd *cobra.Command) { flag.SetDifferentialTransferFlags(cpCmd, true) flag.SetNoRootFlags(cpCmd) flag.SetSyncFlags(cpCmd) + flag.SetTransferReportFlags(getCmd) rootCmd.AddCommand(cpCmd) } func processCpCommand(command *cobra.Command, args []string) error { + cp, err := NewCpCommand(command, args) + if err != nil { + return err + } + + return cp.Process() +} + +type CpCommand struct { + command *cobra.Command + + commonFlagValues *flag.CommonFlagValues + recursiveFlagValues *flag.RecursiveFlagValues + forceFlagValues *flag.ForceFlagValues + progressFlagValues *flag.ProgressFlagValues + retryFlagValues *flag.RetryFlagValues + differentialTransferFlagValues *flag.DifferentialTransferFlagValues + noRootFlagValues *flag.NoRootFlagValues + syncFlagValues *flag.SyncFlagValues + transferReportFlagValues *flag.TransferReportFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + sourcePaths []string + targetPath string + + parallelJobManager *commons.ParallelJobManager + transferReportManager *commons.TransferReportManager + updatedPathMap map[string]bool +} + +func NewCpCommand(command *cobra.Command, args []string) (*CpCommand, error) { + cp := &CpCommand{ + command: command, + + commonFlagValues: flag.GetCommonFlagValues(command), + recursiveFlagValues: flag.GetRecursiveFlagValues(), + forceFlagValues: flag.GetForceFlagValues(), + progressFlagValues: flag.GetProgressFlagValues(), + retryFlagValues: flag.GetRetryFlagValues(), + differentialTransferFlagValues: flag.GetDifferentialTransferFlagValues(), + noRootFlagValues: flag.GetNoRootFlagValues(), + syncFlagValues: flag.GetSyncFlagValues(), + transferReportFlagValues: flag.GetTransferReportFlagValues(command), + + updatedPathMap: map[string]bool{}, + } + + // path + cp.targetPath = args[len(args)-1] + cp.sourcePaths = args[:len(args)-1] + + if cp.noRootFlagValues.NoRoot && len(cp.sourcePaths) > 1 { + return nil, xerrors.Errorf("failed to copy multiple source collections without creating root directory") + } + + return cp, nil +} + +func (cp *CpCommand) Process() error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "processCpCommand", + "struct": "CpCommand", + "function": "Process", }) - cont, err := flag.ProcessCommonFlags(command) + cont, err := flag.ProcessCommonFlags(cp.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -59,67 +125,61 @@ func processCpCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - commonFlagValues := flag.GetCommonFlagValues(command) - recursiveFlagValues := flag.GetRecursiveFlagValues() - forceFlagValues := flag.GetForceFlagValues() - progressFlagValues := flag.GetProgressFlagValues() - retryFlagValues := flag.GetRetryFlagValues() - differentialTransferFlagValues := flag.GetDifferentialTransferFlagValues() - noRootFlagValues := flag.GetNoRootFlagValues() - syncFlagValues := flag.GetSyncFlagValues() - - if retryFlagValues.RetryNumber > 0 && !retryFlagValues.RetryChild { - err = commons.RunWithRetry(retryFlagValues.RetryNumber, retryFlagValues.RetryIntervalSeconds) + // handle retry + if cp.retryFlagValues.RetryNumber > 0 && !cp.retryFlagValues.RetryChild { + err = commons.RunWithRetry(cp.retryFlagValues.RetryNumber, cp.retryFlagValues.RetryIntervalSeconds) if err != nil { - return xerrors.Errorf("failed to run with retry %d: %w", retryFlagValues.RetryNumber, err) + return xerrors.Errorf("failed to run with retry %d: %w", cp.retryFlagValues.RetryNumber, err) } return nil } // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + cp.account = commons.GetAccount() + cp.filesystem, err = commons.GetIRODSFSClient(cp.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer cp.filesystem.Release() - defer filesystem.Release() - - targetPath := args[len(args)-1] - sourcePaths := args[:len(args)-1] - - if noRootFlagValues.NoRoot && len(sourcePaths) > 1 { - return xerrors.Errorf("failed to copy multiple source collections without creating root directory") + // transfer report + cp.transferReportManager, err = commons.NewTransferReportManager(cp.transferReportFlagValues.Report, cp.transferReportFlagValues.ReportPath, cp.transferReportFlagValues.ReportToStdout) + if err != nil { + return xerrors.Errorf("failed to create transfer report manager: %w", err) } + defer cp.transferReportManager.Release() - parallelJobManager := commons.NewParallelJobManager(filesystem, commons.TransferTreadNumDefault, progressFlagValues.ShowProgress, progressFlagValues.ShowFullPath) - parallelJobManager.Start() - - inputPathMap := map[string]bool{} + // parallel job manager + cp.parallelJobManager = commons.NewParallelJobManager(cp.filesystem, commons.TransferTreadNumDefault, cp.progressFlagValues.ShowProgress, cp.progressFlagValues.ShowFullPath) + cp.parallelJobManager.Start() - for _, sourcePath := range sourcePaths { - newTargetDirPath, err := makeCopyTargetDirPath(filesystem, sourcePath, targetPath, noRootFlagValues.NoRoot) + // run + if len(cp.sourcePaths) >= 2 { + // multi-source, target must be a dir + err = cp.ensureTargetIsDir(cp.targetPath) if err != nil { - return xerrors.Errorf("failed to make new target path for copy %s to %s: %w", sourcePath, targetPath, err) + return err } + } - err = copyOne(parallelJobManager, inputPathMap, sourcePath, newTargetDirPath, commonFlagValues, recursiveFlagValues, forceFlagValues, differentialTransferFlagValues) + for _, sourcePath := range cp.sourcePaths { + err = cp.copyOne(sourcePath, cp.targetPath) if err != nil { - return xerrors.Errorf("failed to perform copy %s to %s: %w", sourcePath, targetPath, err) + return xerrors.Errorf("failed to copy %q to %q: %w", sourcePath, cp.targetPath, err) } } - parallelJobManager.DoneScheduling() - err = parallelJobManager.Wait() + cp.parallelJobManager.DoneScheduling() + err = cp.parallelJobManager.Wait() if err != nil { return xerrors.Errorf("failed to perform parallel job: %w", err) } // delete extra - if syncFlagValues.Delete { - logger.Infof("deleting extra files and dirs under %s", targetPath) + if cp.syncFlagValues.Delete { + logger.Infof("deleting extra files and directories under %q", cp.targetPath) - err = copyDeleteExtra(filesystem, inputPathMap, targetPath) + err = cp.deleteExtra(cp.targetPath) if err != nil { return xerrors.Errorf("failed to delete extra files: %w", err) } @@ -128,224 +188,368 @@ func processCpCommand(command *cobra.Command, args []string) error { return nil } -func copyOne(parallelJobManager *commons.ParallelJobManager, inputPathMap map[string]bool, sourcePath string, targetPath string, commonFlagValues *flag.CommonFlagValues, recursiveFlagValues *flag.RecursiveFlagValues, forceFlagValues *flag.ForceFlagValues, differentialTransferFlagValues *flag.DifferentialTransferFlagValues) error { - logger := log.WithFields(log.Fields{ - "package": "subcmd", - "function": "copyOne", - }) +func (cp *CpCommand) ensureTargetIsDir(targetPath string) error { + cwd := commons.GetCWD() + home := commons.GetHomeDir() + zone := commons.GetZone() + targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) + + targetEntry, err := cp.filesystem.Stat(targetPath) + if err != nil { + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + + if !targetEntry.IsDir() { + return commons.NewNotDirError(targetPath) + } + + return nil +} +func (cp *CpCommand) copyOne(sourcePath string, targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - filesystem := parallelJobManager.GetFilesystem() - - sourceEntry, err := filesystem.Stat(sourcePath) + sourceEntry, err := cp.filesystem.Stat(sourcePath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", sourcePath, err) + return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) } - if sourceEntry.Type == irodsclient_fs.FileEntry { - // file - targetFilePath := commons.MakeTargetIRODSFilePath(filesystem, sourcePath, targetPath) - commons.MarkPathMap(inputPathMap, targetFilePath) + if sourceEntry.IsDir() { + // dir + if !cp.recursiveFlagValues.Recursive { + return xerrors.Errorf("cannot copy a collection, turn on 'recurse' option") + } - fileExist := false - targetEntry, err := filesystem.StatFile(targetFilePath) - if err != nil { - if !irodsclient_types.IsFileNotFoundError(err) { - return xerrors.Errorf("failed to stat %s: %w", targetFilePath, err) - } - } else { - fileExist = true + if !cp.noRootFlagValues.NoRoot { + targetPath = commons.MakeTargetIRODSFilePath(cp.filesystem, sourcePath, targetPath) } - copyTask := func(job *commons.ParallelJob) error { - manager := job.GetManager() - fs := manager.GetFilesystem() + return cp.copyDir(sourceEntry, targetPath) + } - job.Progress(0, 1, false) + // file + targetPath = commons.MakeTargetIRODSFilePath(cp.filesystem, sourcePath, targetPath) + return cp.copyFile(sourceEntry, targetPath) +} - logger.Debugf("copying a data object %s to %s", sourcePath, targetFilePath) - err = fs.CopyFileToFile(sourcePath, targetFilePath, true) - if err != nil { - job.Progress(-1, 1, true) - return xerrors.Errorf("failed to copy %s to %s: %w", sourcePath, targetFilePath, err) - } +func (cp *CpCommand) scheduleCopy(sourceEntry *irodsclient_fs.Entry, targetPath string, targetEntry *irodsclient_fs.Entry) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "CpCommand", + "function": "scheduleCopy", + }) + + copyTask := func(job *commons.ParallelJob) error { + manager := job.GetManager() + fs := manager.GetFilesystem() - logger.Debugf("copied a data object %s to %s", sourcePath, targetFilePath) - job.Progress(1, 1, false) - return nil + job.Progress(0, 1, false) + + logger.Debugf("copying a data object %q to %q", sourceEntry.Path, targetPath) + err := fs.CopyFileToFile(sourceEntry.Path, targetPath, true) + if err != nil { + job.Progress(-1, 1, true) + return xerrors.Errorf("failed to copy %q to %q: %w", sourceEntry.Path, targetPath, err) } - if fileExist { - if differentialTransferFlagValues.DifferentialTransfer { - if differentialTransferFlagValues.NoHash { - if targetEntry.Size == sourceEntry.Size { - commons.Printf("skip copying a file %s. The file already exists!\n", targetFilePath) - logger.Debugf("skip copying a file %s. The file already exists!", targetFilePath) - return nil - } - } else { - if targetEntry.Size == sourceEntry.Size { - // compare hash - if len(sourceEntry.CheckSum) > 0 && bytes.Equal(sourceEntry.CheckSum, targetEntry.CheckSum) { - commons.Printf("skip copying a file %s. The file with the same hash already exists!\n", targetFilePath) - logger.Debugf("skip copying a file %s. The file with the same hash already exists!", targetFilePath) - return nil - } - } - } - } else { - if !forceFlagValues.Force { - // ask - overwrite := commons.InputYN(fmt.Sprintf("file %s already exists. Overwrite?", targetFilePath)) - if !overwrite { - commons.Printf("skip copying a file %s. The file already exists!\n", targetFilePath) - logger.Debugf("skip copying a file %s. The file already exists!", targetFilePath) - return nil - } - } - } + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodCopy, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{}, } - parallelJobManager.Schedule(sourcePath, copyTask, 1, progress.UnitsDefault) - logger.Debugf("scheduled a data object copy %s to %s", sourcePath, targetFilePath) - } else { - // dir - if !recursiveFlagValues.Recursive { - return xerrors.Errorf("cannot copy a collection, turn on 'recurse' option") + if targetEntry != nil { + reportFile.DestSize = targetEntry.Size + reportFile.DestChecksum = hex.EncodeToString(targetEntry.CheckSum) } - logger.Debugf("copying a collection %s to %s", sourcePath, targetPath) + cp.transferReportManager.AddFile(reportFile) - entries, err := filesystem.List(sourceEntry.Path) - if err != nil { - return xerrors.Errorf("failed to list dir %s: %w", sourceEntry.Path, err) + logger.Debugf("copied a data object %q to %q", sourceEntry.Path, targetPath) + job.Progress(1, 1, false) + return nil + } + + err := cp.parallelJobManager.Schedule(sourceEntry.Path, copyTask, 1, progress.UnitsDefault) + if err != nil { + return xerrors.Errorf("failed to schedule copy %q to %q: %w", sourceEntry.Path, targetPath, err) + } + + commons.MarkPathMap(cp.updatedPathMap, targetPath) + + logger.Debugf("scheduled a data object copy %q to %q", sourceEntry.Path, targetPath) + + return nil +} + +func (cp *CpCommand) copyFile(sourceEntry *irodsclient_fs.Entry, targetPath string) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "CpCommand", + "function": "copyFile", + }) + + targetEntry, err := cp.filesystem.Stat(targetPath) + if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // target does not exist + // target must be a file with new name + return cp.scheduleCopy(sourceEntry, targetPath, nil) } - for _, entry := range entries { - targetDirPath := targetPath - if entry.Type == irodsclient_fs.DirectoryEntry { - // dir - targetDirPath = commons.MakeTargetIRODSFilePath(filesystem, entry.Path, targetPath) - err = filesystem.MakeDir(targetDirPath, true) - if err != nil { - return xerrors.Errorf("failed to make dir %s: %w", targetDirPath, err) + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + + // target exists + // target must be a file + if targetEntry.IsDir() { + return commons.NewNotFileError(targetPath) + } + + if cp.differentialTransferFlagValues.DifferentialTransfer { + if cp.differentialTransferFlagValues.NoHash { + if targetEntry.Size == sourceEntry.Size { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodCopy, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"differential", "no_hash", "same file size", "skip"}, } + + cp.transferReportManager.AddFile(reportFile) + + commons.Printf("skip copying a file %q to %q. The file already exists!\n", sourceEntry.Path, targetPath) + logger.Debugf("skip copying a file %q to %q. The file already exists!", sourceEntry.Path, targetPath) + return nil } + } else { + if targetEntry.Size == sourceEntry.Size { + // compare hash + if len(sourceEntry.CheckSum) > 0 && bytes.Equal(sourceEntry.CheckSum, targetEntry.CheckSum) { + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodCopy, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"differential", "same checksum", "skip"}, + } - commons.MarkPathMap(inputPathMap, targetDirPath) + cp.transferReportManager.AddFile(reportFile) - err = copyOne(parallelJobManager, inputPathMap, entry.Path, targetDirPath, commonFlagValues, recursiveFlagValues, forceFlagValues, differentialTransferFlagValues) - if err != nil { - return xerrors.Errorf("failed to perform copy %s to %s: %w", entry.Path, targetPath, err) + commons.Printf("skip copying a file %q to %q. The file with the same hash already exists!\n", sourceEntry.Path, targetPath) + logger.Debugf("skip copying a file %q to %q. The file with the same hash already exists!", sourceEntry.Path, targetPath) + return nil + } + } + } + } else { + if !cp.forceFlagValues.Force { + // ask + overwrite := commons.InputYN(fmt.Sprintf("file %q already exists. Overwrite?", targetPath)) + if !overwrite { + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodCopy, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"no_overwrite", "skip"}, + } + + cp.transferReportManager.AddFile(reportFile) + + commons.Printf("skip copying a file %q to %q. The file already exists!\n", sourceEntry.Path, targetPath) + logger.Debugf("skip copying a file %q to %q. The file already exists!", sourceEntry.Path, targetPath) + return nil } } } - return nil + + // schedule + return cp.scheduleCopy(sourceEntry, targetPath, targetEntry) } -func makeCopyTargetDirPath(filesystem *irodsclient_fs.FileSystem, sourcePath string, targetPath string, noRoot bool) (string, error) { - cwd := commons.GetCWD() - home := commons.GetHomeDir() - zone := commons.GetZone() - sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) - targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) +func (cp *CpCommand) copyDir(sourceEntry *irodsclient_fs.Entry, targetPath string) error { + targetEntry, err := cp.filesystem.Stat(targetPath) + if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // target does not exist + // target must be a directory with new name + err = cp.filesystem.MakeDir(targetPath, true) + if err != nil { + return xerrors.Errorf("failed to make a directory %q: %w", targetPath, err) + } + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodCopy, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + DestPath: targetPath, + Notes: []string{"directory"}, + } + + cp.transferReportManager.AddFile(reportFile) + } else { + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + } else { + // target exists + if !targetEntry.IsDir() { + return commons.NewNotDirError(targetPath) + } + } - sourceEntry, err := filesystem.Stat(sourcePath) + // copy entries + entries, err := cp.filesystem.List(sourceEntry.Path) if err != nil { - return "", xerrors.Errorf("failed to stat %s: %w", sourcePath, err) + return xerrors.Errorf("failed to list a directory %q: %w", sourceEntry.Path, err) } - if sourceEntry.Type == irodsclient_fs.FileEntry { - // file - targetFilePath := commons.MakeTargetIRODSFilePath(filesystem, sourcePath, targetPath) - targetDirPath := commons.GetDir(targetFilePath) - _, err := filesystem.Stat(targetDirPath) - if err != nil { - return "", xerrors.Errorf("failed to stat dir %s: %w", targetDirPath, err) - } + for _, entry := range entries { + newEntryPath := commons.MakeTargetIRODSFilePath(cp.filesystem, entry.Path, targetPath) - return targetDirPath, nil - } else { - // dir - targetDirPath := targetPath - - if filesystem.ExistsDir(targetDirPath) { - // already exist - if !noRoot { - targetDirPath = commons.MakeTargetIRODSFilePath(filesystem, sourcePath, targetDirPath) - err = filesystem.MakeDir(targetDirPath, true) - if err != nil { - return "", xerrors.Errorf("failed to make dir %s: %w", targetDirPath, err) - } + if entry.IsDir() { + // dir + err = cp.copyDir(entry, newEntryPath) + if err != nil { + return err } } else { - err = filesystem.MakeDir(targetDirPath, true) + // file + err = cp.copyFile(entry, newEntryPath) if err != nil { - return "", xerrors.Errorf("failed to make dir %s: %w", targetDirPath, err) + return err } } - return targetDirPath, nil + commons.MarkPathMap(cp.updatedPathMap, newEntryPath) } + + commons.MarkPathMap(cp.updatedPathMap, targetPath) + + return nil } -func copyDeleteExtra(filesystem *irodsclient_fs.FileSystem, inputPathMap map[string]bool, targetPath string) error { +func (cp *CpCommand) deleteExtra(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - return copyDeleteExtraInternal(filesystem, inputPathMap, targetPath) + return cp.deleteExtraInternal(targetPath) } -func copyDeleteExtraInternal(filesystem *irodsclient_fs.FileSystem, inputPathMap map[string]bool, targetPath string) error { +func (cp *CpCommand) deleteExtraInternal(targetPath string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "copyDeleteExtraInternal", + "struct": "CpCommand", + "function": "deleteExtraInternal", }) - targetEntry, err := filesystem.Stat(targetPath) + targetEntry, err := cp.filesystem.Stat(targetPath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", targetPath, err) + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } - if targetEntry.Type == irodsclient_fs.FileEntry { - if _, ok := inputPathMap[targetPath]; !ok { + // target is file + if !targetEntry.IsDir() { + if _, ok := cp.updatedPathMap[targetPath]; !ok { // extra file - logger.Debugf("removing an extra data object %s", targetPath) - removeErr := filesystem.RemoveFile(targetPath, true) - if removeErr != nil { - return removeErr + logger.Debugf("removing an extra data object %q", targetPath) + + removeErr := cp.filesystem.RemoveFile(targetPath, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"extra", "cp"}, } - } - } else { - // dir - if _, ok := inputPathMap[targetPath]; !ok { - // extra dir - logger.Debugf("removing an extra collection %s", targetPath) - removeErr := filesystem.RemoveDir(targetPath, true, true) + + cp.transferReportManager.AddFile(reportFile) + if removeErr != nil { return removeErr } - } else { - // non extra dir - entries, err := filesystem.List(targetPath) - if err != nil { - return xerrors.Errorf("failed to list dir %s: %w", targetPath, err) - } + } - for idx := range entries { - newTargetPath := entries[idx].Path + return nil + } - err = copyDeleteExtraInternal(filesystem, inputPathMap, newTargetPath) - if err != nil { - return err - } + // target is dir + if _, ok := cp.updatedPathMap[targetPath]; !ok { + // extra dir + logger.Debugf("removing an extra collection %q", targetPath) + + removeErr := cp.filesystem.RemoveDir(targetPath, true, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"extra", "cp", "dir"}, + } + + cp.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + // non extra dir + // scan recursively + entries, err := cp.filesystem.List(targetPath) + if err != nil { + return xerrors.Errorf("failed to list a directory %q: %w", targetPath, err) + } + + for _, entry := range entries { + newTargetPath := path.Join(targetPath, entry.Name) + err = cp.deleteExtraInternal(newTargetPath) + if err != nil { + return err } } } diff --git a/cmd/subcmd/env.go b/cmd/subcmd/env.go index 1029147..fcfa0fd 100644 --- a/cmd/subcmd/env.go +++ b/cmd/subcmd/env.go @@ -1,8 +1,11 @@ package subcmd import ( + "os" + "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" + "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" "golang.org/x/xerrors" ) @@ -24,7 +27,27 @@ func AddEnvCommand(rootCmd *cobra.Command) { } func processEnvCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + bun, err := NewBunCommand(command, args) + if err != nil { + return err + } + + return bun.Process() +} + +type EnvCommand struct { + command *cobra.Command +} + +func NewEnvCommand(command *cobra.Command, args []string) (*EnvCommand, error) { + env := &EnvCommand{ + command: command, + } + + return env, nil +} +func (env *EnvCommand) Process() error { + cont, err := flag.ProcessCommonFlags(env.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -33,9 +56,94 @@ func processEnvCommand(command *cobra.Command, args []string) error { return nil } - err = commons.PrintEnvironment() + err = env.printEnvironment() if err != nil { return xerrors.Errorf("failed to print environment: %w", err) } + + return nil +} + +func (env *EnvCommand) printEnvironment() error { + envMgr := commons.GetEnvironmentManager() + if envMgr == nil { + return xerrors.Errorf("environment is not set") + } + + t := table.NewWriter() + t.SetOutputMirror(os.Stdout) + + t.AppendRows([]table.Row{ + { + "iRODS Session Environment File", + envMgr.GetSessionFilePath(os.Getppid()), + }, + { + "iRODS Environment File", + envMgr.GetEnvironmentFilePath(), + }, + { + "iRODS Host", + envMgr.Environment.Host, + }, + { + "iRODS Port", + envMgr.Environment.Port, + }, + { + "iRODS Zone", + envMgr.Environment.Zone, + }, + { + "iRODS Username", + envMgr.Environment.Username, + }, + { + "iRODS Default Resource", + envMgr.Environment.DefaultResource, + }, + { + "iRODS Default Hash Scheme", + envMgr.Environment.DefaultHashScheme, + }, + { + "iRODS Authentication Scheme", + envMgr.Environment.AuthenticationScheme, + }, + { + "iRODS Client Server Negotiation", + envMgr.Environment.ClientServerNegotiation, + }, + { + "iRODS Client Server Policy", + envMgr.Environment.ClientServerPolicy, + }, + { + "iRODS SSL CA Certification File", + envMgr.Environment.SSLCACertificateFile, + }, + { + "iRODS SSL CA Certification Path", + envMgr.Environment.SSLCACertificatePath, + }, + { + "iRODS SSL Encryption Key Size", + envMgr.Environment.EncryptionKeySize, + }, + { + "iRODS SSL Encryption Key Algorithm", + envMgr.Environment.EncryptionAlgorithm, + }, + { + "iRODS SSL Encryption Salt Size", + envMgr.Environment.EncryptionSaltSize, + }, + { + "iRODS SSL Encryption Hash Rounds", + envMgr.Environment.EncryptionNumHashRounds, + }, + }, table.RowConfig{}) + t.Render() + return nil } diff --git a/cmd/subcmd/get.go b/cmd/subcmd/get.go index f669d1c..d8c50c0 100644 --- a/cmd/subcmd/get.go +++ b/cmd/subcmd/get.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "os" + "path" "time" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" @@ -48,6 +49,15 @@ func AddGetCommand(rootCmd *cobra.Command) { rootCmd.AddCommand(getCmd) } +func processGetCommand(command *cobra.Command, args []string) error { + get, err := NewGetCommand(command, args) + if err != nil { + return err + } + + return get.Process() +} + type GetCommand struct { command *cobra.Command @@ -74,7 +84,7 @@ type GetCommand struct { parallelJobManager *commons.ParallelJobManager transferReportManager *commons.TransferReportManager - inputPathMap map[string]bool + updatedPathMap map[string]bool } func NewGetCommand(command *cobra.Command, args []string) (*GetCommand, error) { @@ -94,14 +104,14 @@ func NewGetCommand(command *cobra.Command, args []string) (*GetCommand, error) { postTransferFlagValues: flag.GetPostTransferFlagValues(), transferReportFlagValues: flag.GetTransferReportFlagValues(command), - inputPathMap: map[string]bool{}, + updatedPathMap: map[string]bool{}, } get.maxConnectionNum = get.parallelTransferFlagValues.ThreadNumber + 2 // 2 for metadata op // path get.targetPath = "./" - get.sourcePaths = args[:] + get.sourcePaths = args if len(args) >= 2 { get.targetPath = args[len(args)-1] @@ -141,7 +151,7 @@ func (get *GetCommand) Process() error { appConfig := commons.GetConfig() syncAccount := false if len(get.ticketAccessFlagValues.Name) > 0 { - logger.Debugf("use ticket: %s", get.ticketAccessFlagValues.Name) + logger.Debugf("use ticket %q", get.ticketAccessFlagValues.Name) appConfig.Ticket = get.ticketAccessFlagValues.Name syncAccount = true } @@ -159,7 +169,6 @@ func (get *GetCommand) Process() error { if err != nil { return xerrors.Errorf("failed to run with retry %d: %w", get.retryFlagValues.RetryNumber, err) } - return nil } @@ -169,7 +178,6 @@ func (get *GetCommand) Process() error { if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } - defer get.filesystem.Release() // transfer report @@ -177,7 +185,6 @@ func (get *GetCommand) Process() error { if err != nil { return xerrors.Errorf("failed to create transfer report manager: %w", err) } - defer get.transferReportManager.Release() // set default key for decryption @@ -185,24 +192,23 @@ func (get *GetCommand) Process() error { get.decryptionFlagValues.Key = get.account.Password } - if get.noRootFlagValues.NoRoot && len(get.sourcePaths) > 1 { - return xerrors.Errorf("failed to get multiple source collections without creating root directory") - } - // parallel job manager get.parallelJobManager = commons.NewParallelJobManager(get.filesystem, get.parallelTransferFlagValues.ThreadNumber, get.progressFlagValues.ShowProgress, get.progressFlagValues.ShowFullPath) get.parallelJobManager.Start() // run - for _, sourcePath := range get.sourcePaths { - newSourcePath, newTargetDirPath, err := get.makeSourceAndTargetDirPath(sourcePath, get.targetPath) + if len(get.sourcePaths) >= 2 { + // multi-source, target must be a dir + err = get.ensureTargetIsDir(get.targetPath) if err != nil { - return xerrors.Errorf("failed to make new target path for get %s to %s: %w", sourcePath, get.targetPath, err) + return err } + } - err = get.getOne(newSourcePath, newTargetDirPath, get.decryptionFlagValues.Decryption) + for _, sourcePath := range get.sourcePaths { + err = get.getOne(sourcePath, get.targetPath) if err != nil { - return xerrors.Errorf("failed to perform get %s to %s: %w", newSourcePath, newTargetDirPath, err) + return xerrors.Errorf("failed to get %q to %q: %w", sourcePath, get.targetPath, err) } } @@ -212,258 +218,57 @@ func (get *GetCommand) Process() error { return xerrors.Errorf("failed to perform parallel jobs: %w", err) } - return nil -} - -func (get *GetCommand) makeSourceAndTargetDirPath(sourcePath string, targetPath string) (string, string, error) { - cwd := commons.GetCWD() - home := commons.GetHomeDir() - zone := commons.GetZone() - sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) - targetPath = commons.MakeLocalPath(targetPath) - - sourceEntry, err := get.filesystem.Stat(sourcePath) - if err != nil { - return "", "", xerrors.Errorf("failed to stat %s: %w", sourcePath, err) - } + // delete on success + if get.postTransferFlagValues.DeleteOnSuccess { + for _, sourcePath := range get.sourcePaths { + logger.Infof("deleting source %q after successful data get", sourcePath) - if sourceEntry.Type == irodsclient_fs.FileEntry { - // file - targetFilePath := commons.MakeTargetLocalFilePath(sourcePath, targetPath) - targetDirPath := commons.GetDir(targetFilePath) - _, err := os.Stat(targetDirPath) - if err != nil { - if os.IsNotExist(err) { - return "", "", irodsclient_types.NewFileNotFoundError(targetDirPath) + err := get.deleteOnSuccess(sourcePath) + if err != nil { + return xerrors.Errorf("failed to delete source %q: %w", sourcePath, err) } - - return "", "", xerrors.Errorf("failed to stat dir %s: %w", targetDirPath, err) } - - return sourcePath, targetDirPath, nil } - // dir - _, err = os.Stat(targetPath) - if err != nil { - if os.IsNotExist(err) { - return "", "", irodsclient_types.NewFileNotFoundError(targetPath) - } - - return "", "", xerrors.Errorf("failed to stat dir %s: %w", targetPath, err) - } - - targetDirPath := targetPath + // delete extra + if get.syncFlagValues.Delete { + logger.Infof("deleting extra files and directories under %q", get.targetPath) - if !get.noRootFlagValues.NoRoot { - // make target dir - targetDirPath = commons.MakeTargetLocalFilePath(sourceEntry.Path, targetDirPath) - err = os.MkdirAll(targetDirPath, 0766) + err := get.deleteExtra(get.targetPath) if err != nil { - return "", "", xerrors.Errorf("failed to make dir %s: %w", targetDirPath, err) + return xerrors.Errorf("failed to delete extra files: %w", err) } } - return sourcePath, targetDirPath, nil -} - -func (get *GetCommand) getEncryptionManagerForDecryption(mode commons.EncryptionMode) *commons.EncryptionManager { - manager := commons.NewEncryptionManager(mode) - - switch mode { - case commons.EncryptionModeWinSCP, commons.EncryptionModePGP: - manager.SetKey([]byte(get.decryptionFlagValues.Key)) - case commons.EncryptionModeSSH: - manager.SetPublicPrivateKey(get.decryptionFlagValues.PrivateKeyPath) - } - - return manager + return nil } -func (get *GetCommand) getOne(sourcePath string, targetPath string, requireDecryption bool) error { - logger := log.WithFields(log.Fields{ - "package": "subcmd", - "struct": "GetCommand", - "function": "getOne", - }) +func (get *GetCommand) ensureTargetIsDir(targetPath string) error { + targetPath = commons.MakeLocalPath(targetPath) - sourceEntry, err := get.filesystem.Stat(sourcePath) + targetStat, err := os.Stat(targetPath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", sourcePath, err) + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } - // load encryption config - if !requireDecryption { - requireDecryption = get.requireDecryptionByMeta(sourceEntry) - } - - // if source is dir, recursive - if sourceEntry.Type == irodsclient_fs.DirectoryEntry { - // dir - logger.Debugf("downloading a collection %s to %s", sourcePath, targetPath) - - entries, err := filesystem.List(sourceEntry.Path) - if err != nil { - return xerrors.Errorf("failed to list dir %s: %w", sourceEntry.Path, err) - } - - for _, entry := range entries { - targetDirPath := targetPath - if entry.Type != irodsclient_fs.FileEntry { - // dir - targetDirPath = commons.MakeTargetLocalFilePath(entry.Path, targetPath) - err = os.MkdirAll(targetDirPath, 0766) - if err != nil { - return xerrors.Errorf("failed to make dir %s: %w", targetDirPath, err) - } - } - - commons.MarkPathMap(get.inputPathMap, targetDirPath) - - err = get.getOne(entry.Path, targetDirPath, requireDecryption) - if err != nil { - return xerrors.Errorf("failed to perform get %s to %s: %w", entry.Path, targetDirPath, err) - } - } - - return nil - } - - if sourceEntry.Type != irodsclient_fs.FileEntry { - return xerrors.Errorf("unhandled file entry type %s", sourceEntry.Type) - } - - // file - - commons.MarkPathMap(get.inputPathMap, decryptedTargetFilePath) - - fileExist := false - targetEntry, err := os.Stat(targetFilePath) - if err != nil { - if !os.IsNotExist(err) { - return xerrors.Errorf("failed to stat %s: %w", targetFilePath, err) - } - } else { - fileExist = true + if !targetStat.IsDir() { + return commons.NewNotDirError(targetPath) } - getTask := get.getTask(sourcePath, targetFilePath, sourceEntry.Size) - - if fileExist { - // check transfer status file - trxStatusFilePath := irodsclient_irodsfs.GetDataObjectTransferStatusFilePath(targetFilePath) - trxStatusFileExist := false - _, err = os.Stat(trxStatusFilePath) - if err == nil { - trxStatusFileExist = true - } - - if trxStatusFileExist { - // incomplete file - resume downloading - commons.Printf("resume downloading a data object %s\n", targetFilePath) - logger.Debugf("resume downloading a data object %s", targetFilePath) - } else if get.differentialTransferFlagValues.DifferentialTransfer { - // trx status not exist - if get.differentialTransferFlagValues.NoHash { - if targetEntry.Size() == sourceEntry.Size { - // skip - now := time.Now() - reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodGet, - StartAt: now, - EndAt: now, - LocalPath: targetFilePath, - LocalSize: targetEntry.Size(), - IrodsPath: sourcePath, - IrodsSize: sourceEntry.Size, - IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"differential", "no_hash", "same file size", "skip"}, - } - - get.transferReportManager.AddFile(reportFile) - - commons.Printf("skip downloading a data object %s. The file already exists!\n", targetFilePath) - logger.Debugf("skip downloading a data object %s. The file already exists!", targetFilePath) - return nil - } - } else { - if targetEntry.Size() == sourceEntry.Size { - if len(sourceEntry.CheckSum) > 0 { - // compare hash - hash, err := irodsclient_util.HashLocalFile(targetFilePath, string(sourceEntry.CheckSumAlgorithm)) - if err != nil { - return xerrors.Errorf("failed to get hash of %s: %w", targetFilePath, err) - } - - if bytes.Equal(sourceEntry.CheckSum, hash) { - // skip - now := time.Now() - reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodGet, - StartAt: now, - EndAt: now, - LocalPath: targetFilePath, - LocalSize: targetEntry.Size(), - IrodsPath: sourcePath, - IrodsSize: sourceEntry.Size, - IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"differential", "same hash", "same file size", "skip"}, - } - - get.transferReportManager.AddFile(reportFile) - - commons.Printf("skip downloading a data object %s. The file with the same hash already exists!\n", targetFilePath) - logger.Debugf("skip downloading a data object %s. The file with the same hash already exists!", targetFilePath) - return nil - } - } - } - } - } else { - if !get.forceFlagValues.Force { - // ask - overwrite := commons.InputYN(fmt.Sprintf("file %s already exists. Overwrite?", targetFilePath)) - if !overwrite { - // skip - now := time.Now() - reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodGet, - StartAt: now, - EndAt: now, - LocalPath: targetFilePath, - LocalSize: targetEntry.Size(), - IrodsPath: sourcePath, - IrodsSize: sourceEntry.Size, - IrodsChecksum: hex.EncodeToString(sourceEntry.CheckSum), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"no overwrite", "skip"}, - } - - get.transferReportManager.AddFile(reportFile) + return nil +} - commons.Printf("skip downloading a data object %s. The file already exists!\n", targetFilePath) - logger.Debugf("skip downloading a data object %s. The file already exists!", targetFilePath) - return nil - } - } - } +func (get *GetCommand) requireDecryption(sourceEntry *irodsclient_fs.Entry, parentDecryption bool) bool { + if get.decryptionFlagValues.Decryption { + return true } - threadsRequired := irodsclient_util.GetNumTasksForParallelTransfer(sourceEntry.Size) - err = get.parallelJobManager.Schedule(sourcePath, getTask, threadsRequired, progress.UnitsBytes) - if err != nil { - return xerrors.Errorf("failed to schedule %s: %w", sourcePath, err) + if get.decryptionFlagValues.NoDecryption { + return false } - logger.Debugf("scheduled a data object download %s to %s", sourcePath, targetPath) - return nil -} - -func (get *GetCommand) requireDecryptionByMeta(sourceEntry *irodsclient_fs.Entry) bool { - // load encryption config from meta - if !get.decryptionFlagValues.NoDecryption && !get.decryptionFlagValues.IgnoreMeta { + if !get.decryptionFlagValues.IgnoreMeta { + // load encryption config from meta sourceDir := sourceEntry.Path if !sourceEntry.IsDir() { sourceDir = commons.GetDir(sourceEntry.Path) @@ -471,79 +276,68 @@ func (get *GetCommand) requireDecryptionByMeta(sourceEntry *irodsclient_fs.Entry encryptionConfig := commons.GetEncryptionConfigFromMeta(get.filesystem, sourceDir) - if encryptionConfig.Required { - return encryptionConfig.Required - } + return encryptionConfig.Required } - return false + return parentDecryption } -func (get *GetCommand) requireDecryption(sourcePath string) bool { - encryptionMode := commons.DetectEncryptionMode(sourcePath) - if get.decryptionFlagValues.Decryption && encryptionMode != commons.EncryptionModeUnknown { +func (get *GetCommand) hasTransferStatusFile(targetPath string) bool { + // check transfer status file + trxStatusFilePath := irodsclient_irodsfs.GetDataObjectTransferStatusFilePath(targetPath) + _, err := os.Stat(trxStatusFilePath) + if err == nil { return true } return false } -func (get *GetCommand) getPathsForDecryption(sourcePath string, targetPath string) (string, string, error) { - sourceFilename := commons.GetBasename(sourcePath) - - encryptionMode := commons.DetectEncryptionMode(sourceFilename) - encryptManager := get.getEncryptionManagerForDecryption(encryptionMode) +func (get *GetCommand) getOne(sourcePath string, targetPath string) error { + cwd := commons.GetCWD() + home := commons.GetHomeDir() + zone := commons.GetZone() + sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) + targetPath = commons.MakeLocalPath(targetPath) - if get.requireDecryption(sourcePath) { - tempFilePath := commons.MakeTargetLocalFilePath(sourcePath, get.decryptionFlagValues.TempPath) + sourceEntry, err := get.filesystem.Stat(sourcePath) + if err != nil { + return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) + } - decryptedFilename, err := encryptManager.DecryptFilename(sourceFilename) - if err != nil { - return "", "", xerrors.Errorf("failed to decrypt filename %s: %w", sourcePath, err) + if sourceEntry.IsDir() { + // dir + if !get.noRootFlagValues.NoRoot { + targetPath = commons.MakeTargetLocalFilePath(sourcePath, targetPath) } - targetFilePath := commons.MakeTargetLocalFilePath(decryptedFilename, targetPath) - - return tempFilePath, targetFilePath, nil + return get.getDir(sourceEntry, targetPath, false) } - targetFilePath := commons.MakeTargetLocalFilePath(sourcePath, targetPath) - - return "", targetFilePath, nil -} - -func (get *GetCommand) decryptFile(sourcePath string, encryptedFilePath string, targetPath string) error { - logger := log.WithFields(log.Fields{ - "package": "subcmd", - "struct": "GetCommand", - "function": "decryptFile", - }) - - encryptionMode := commons.DetectEncryptionMode(sourcePath) - encryptManager := get.getEncryptionManagerForDecryption(encryptionMode) - - if get.requireDecryption(sourcePath) { - logger.Debugf("decrypt a data object %s to %s", encryptedFilePath, targetPath) - err := encryptManager.DecryptFile(encryptedFilePath, targetPath) + // file + requireDecryption := get.requireDecryption(sourceEntry, false) + if requireDecryption { + // decrypt filename + tempPath, newTargetPath, err := get.getPathsForDecryption(sourceEntry.Path, targetPath) if err != nil { - return xerrors.Errorf("failed to decrypt %s: %w", encryptedFilePath, err) + return xerrors.Errorf("failed to get decryption path for %q: %w", sourceEntry.Path, err) } - logger.Debugf("removing a temp file %s", encryptedFilePath) - os.Remove(encryptedFilePath) + return get.getFile(sourceEntry, tempPath, newTargetPath, requireDecryption) } - return nil + targetPath = commons.MakeTargetLocalFilePath(sourcePath, targetPath) + return get.getFile(sourceEntry, "", targetPath, requireDecryption) } -func (get *GetCommand) getTask(sourcePath string, targetPath string, sourceSize int64) func(job *commons.ParallelJob) error { +func (get *GetCommand) scheduleGet(sourceEntry *irodsclient_fs.Entry, tempPath string, targetPath string, resume bool, requireDecryption bool) error { logger := log.WithFields(log.Fields{ "package": "subcmd", "struct": "GetCommand", - "function": "getTask", + "function": "scheduleGet", }) - return func(job *commons.ParallelJob) error { + getTask := func(job *commons.ParallelJob) error { manager := job.GetManager() fs := manager.GetFilesystem() @@ -551,88 +345,322 @@ func (get *GetCommand) getTask(sourcePath string, targetPath string, sourceSize job.Progress(processed, total, false) } - job.Progress(0, sourceSize, false) + job.Progress(0, sourceEntry.Size, false) - logger.Debugf("downloading a data object %s to %s", sourcePath, targetPath) + logger.Debugf("downloading a data object %q to %q", sourceEntry.Path, targetPath) var downloadErr error var downloadResult *irodsclient_fs.FileTransferResult notes := []string{} + downloadPath := targetPath + if len(tempPath) > 0 { + downloadPath = tempPath + } + // determine how to download if get.parallelTransferFlagValues.SingleTread || get.parallelTransferFlagValues.ThreadNumber == 1 { - downloadResult, downloadErr = fs.DownloadFileResumable(sourcePath, "", targetPath, get.checksumFlagValues.VerifyChecksum, callbackGet) - notes = append(notes, "icat") - notes = append(notes, "single-thread") + downloadResult, downloadErr = fs.DownloadFileResumable(sourceEntry.Path, "", downloadPath, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "icat", "single-thread") } else if get.parallelTransferFlagValues.RedirectToResource { - downloadResult, downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) - notes = append(notes, "redirect-to-resource") + if resume { + downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourceEntry.Path, "", downloadPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "icat", "multi-thread", "resume") + } else { + downloadResult, downloadErr = fs.DownloadFileRedirectToResource(sourceEntry.Path, "", downloadPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "redirect-to-resource") + } } else if get.parallelTransferFlagValues.Icat { - downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) - notes = append(notes, "icat") - notes = append(notes, "multi-thread") + downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourceEntry.Path, "", downloadPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "icat", "multi-thread") } else { // auto - if sourceSize >= commons.RedirectToResourceMinSize { + if sourceEntry.Size >= commons.RedirectToResourceMinSize { // redirect-to-resource - downloadResult, downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) - notes = append(notes, "redirect-to-resource") + if resume { + downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourceEntry.Path, "", downloadPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "icat", "multi-thread", "resume") + } else { + downloadResult, downloadErr = fs.DownloadFileRedirectToResource(sourceEntry.Path, "", downloadPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "redirect-to-resource") + } } else { - downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) - notes = append(notes, "icat") - notes = append(notes, "multi-thread") + downloadResult, downloadErr = fs.DownloadFileParallelResumable(sourceEntry.Path, "", downloadPath, 0, get.checksumFlagValues.VerifyChecksum, callbackGet) + notes = append(notes, "icat", "multi-thread") } } - get.transferReportManager.AddTransfer(downloadResult, commons.TransferMethodGet, downloadErr, notes) - if downloadErr != nil { - job.Progress(-1, sourceSize, true) - return xerrors.Errorf("failed to download %s to %s: %w", sourcePath, targetPath, downloadErr) + job.Progress(-1, sourceEntry.Size, true) + return xerrors.Errorf("failed to download %q to %q: %w", sourceEntry.Path, targetPath, downloadErr) } - logger.Debugf("downloaded a data object %s to %s", sourcePath, targetPath) - job.Progress(sourceSize, sourceSize, false) + // decrypt + if requireDecryption { + decrypted, err := get.decryptFile(sourceEntry.Path, tempPath, targetPath) + if err != nil { + job.Progress(-1, sourceEntry.Size, true) + return xerrors.Errorf("failed to decrypt file: %w", err) + } + + if decrypted { + notes = append(notes, "decrypted", targetPath) + } + } + + err := get.transferReportManager.AddTransfer(downloadResult, commons.TransferMethodGet, downloadErr, notes) + if err != nil { + job.Progress(-1, sourceEntry.Size, true) + return xerrors.Errorf("failed to add transfer report: %w", err) + } + + logger.Debugf("downloaded a data object %q to %q", sourceEntry.Path, targetPath) + job.Progress(sourceEntry.Size, sourceEntry.Size, false) return nil } + + threadsRequired := irodsclient_util.GetNumTasksForParallelTransfer(sourceEntry.Size) + err := get.parallelJobManager.Schedule(sourceEntry.Path, getTask, threadsRequired, progress.UnitsBytes) + if err != nil { + return xerrors.Errorf("failed to schedule download %q to %q: %w", sourceEntry.Path, targetPath, err) + } + + commons.MarkPathMap(get.updatedPathMap, targetPath) + + logger.Debugf("scheduled a data object download %q to %q", sourceEntry.Path, targetPath) + + return nil } -func (get *GetCommand) deleteOnSuccess(sourcePath string) error { +func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath string, targetPath string, requireDecryption bool) error { logger := log.WithFields(log.Fields{ "package": "subcmd", "struct": "GetCommand", - "function": "deleteOnSuccess", + "function": "getFile", }) - if get.postTransferFlagValues.DeleteOnSuccess { - logger.Debugf("removing source file %s", sourcePath) - get.filesystem.RemoveFile(sourcePath, true) + targetStat, err := os.Stat(targetPath) + if err != nil { + if os.IsNotExist(err) { + // target does not exist + // target must be a file with new name + return get.scheduleGet(sourceEntry, tempPath, targetPath, false, requireDecryption) + } + + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } - return nil + // target exists + // target must be a file + if targetStat.IsDir() { + return commons.NewNotFileError(targetPath) + } + + // check transfer status file + if get.hasTransferStatusFile(targetPath) { + // incomplete file - resume downloading + commons.Printf("resume downloading a data object %q\n", targetPath) + logger.Debugf("resume downloading a data object %q", targetPath) + + return get.scheduleGet(sourceEntry, tempPath, targetPath, true, requireDecryption) + } + + if get.differentialTransferFlagValues.DifferentialTransfer { + if get.differentialTransferFlagValues.NoHash { + if targetStat.Size() == sourceEntry.Size { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + + DestPath: targetPath, + DestSize: targetStat.Size(), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"differential", "no_hash", "same file size", "skip"}, + } + + get.transferReportManager.AddFile(reportFile) + + commons.Printf("skip downloading a data object %q to %q. The file already exists!\n", sourceEntry.Path, targetPath) + logger.Debugf("skip downloading a data object %q to %q. The file already exists!", sourceEntry.Path, targetPath) + return nil + } + } else { + if targetStat.Size() == sourceEntry.Size { + // compare hash + if len(sourceEntry.CheckSum) > 0 { + localChecksum, err := irodsclient_util.HashLocalFile(targetPath, string(sourceEntry.CheckSumAlgorithm)) + if err != nil { + return xerrors.Errorf("failed to get hash of %q: %w", targetPath, err) + } + + if bytes.Equal(sourceEntry.CheckSum, localChecksum) { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + DestSize: targetStat.Size(), + DestChecksum: hex.EncodeToString(localChecksum), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"differential", "same checksum", "skip"}, + } + + get.transferReportManager.AddFile(reportFile) + + commons.Printf("skip downloading a data object %q to %q. The file with the same hash already exists!\n", sourceEntry.Path, targetPath) + logger.Debugf("skip downloading a data object %q to %q. The file with the same hash already exists!", sourceEntry.Path, targetPath) + return nil + } + } + } + } + } else { + if !get.forceFlagValues.Force { + // ask + overwrite := commons.InputYN(fmt.Sprintf("file %q already exists. Overwrite?", targetPath)) + if !overwrite { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + DestSize: targetStat.Size(), + ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"no_overwrite", "skip"}, + } + + get.transferReportManager.AddFile(reportFile) + + commons.Printf("skip downloading a data object %q to %q. The file already exists!\n", sourceEntry.Path, targetPath) + logger.Debugf("skip downloading a data object %q to %q. The file already exists!", sourceEntry.Path, targetPath) + return nil + } + } + } + + // schedule + return get.scheduleGet(sourceEntry, tempPath, targetPath, false, requireDecryption) } -func (get *GetCommand) deleteExtra() error { - logger := log.WithFields(log.Fields{ - "package": "subcmd", - "struct": "GetCommand", - "function": "deleteExtra", - }) +func (get *GetCommand) getDir(sourceEntry *irodsclient_fs.Entry, targetPath string, parentDecryption bool) error { + targetStat, err := os.Stat(targetPath) + if err != nil { + if os.IsNotExist(err) { + // target does not exist + // target must be a directorywith new name + err = os.MkdirAll(targetPath, 0766) + if err != nil { + return xerrors.Errorf("failed to make a directory %q: %w", targetPath, err) + } - if get.syncFlagValues.Delete { - logger.Infof("deleting extra files and dirs under %s", get.targetPath) - targetPath := commons.MakeLocalPath(get.targetPath) + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + DestPath: targetPath, + Notes: []string{"directory"}, + } - err := get.deleteExtraInternal(targetPath) - if err != nil { - return xerrors.Errorf("failed to delete extra files: %w", err) + get.transferReportManager.AddFile(reportFile) + } else { + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + } else { + // target exists + if !targetStat.IsDir() { + return commons.NewNotDirError(targetPath) } } + // load encryption config + requireDecryption := get.requireDecryption(sourceEntry, parentDecryption) + + // get entries + entries, err := get.filesystem.List(sourceEntry.Path) + if err != nil { + return xerrors.Errorf("failed to list a directory %q: %w", sourceEntry.Path, err) + } + + for _, entry := range entries { + newEntryPath := commons.MakeTargetLocalFilePath(entry.Path, targetPath) + + if entry.IsDir() { + // dir + err = get.getDir(entry, newEntryPath, requireDecryption) + if err != nil { + return err + } + + commons.MarkPathMap(get.updatedPathMap, newEntryPath) + } else { + // file + if requireDecryption { + // decrypt filename + tempPath, newTargetPath, err := get.getPathsForDecryption(entry.Path, targetPath) + if err != nil { + return xerrors.Errorf("failed to get decryption path for %q: %w", entry.Path, err) + } + + err = get.getFile(entry, tempPath, newTargetPath, requireDecryption) + if err != nil { + return err + } + + commons.MarkPathMap(get.updatedPathMap, newTargetPath) + } else { + err = get.getFile(entry, "", newEntryPath, requireDecryption) + if err != nil { + return err + } + + commons.MarkPathMap(get.updatedPathMap, newEntryPath) + } + } + } + + commons.MarkPathMap(get.updatedPathMap, targetPath) + return nil } +func (get *GetCommand) deleteOnSuccess(sourcePath string) error { + sourceEntry, err := get.filesystem.Stat(sourcePath) + if err != nil { + return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) + } + + if sourceEntry.IsDir() { + return get.filesystem.RemoveDir(sourcePath, true, true) + } + + return get.filesystem.RemoveFile(sourcePath, true) +} + +func (get *GetCommand) deleteExtra(targetPath string) error { + targetPath = commons.MakeLocalPath(targetPath) + + return get.deleteExtraInternal(targetPath) +} + func (get *GetCommand) deleteExtraInternal(targetPath string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", @@ -640,37 +668,31 @@ func (get *GetCommand) deleteExtraInternal(targetPath string) error { "function": "deleteExtraInternal", }) - realTargetPath, err := commons.ResolveSymlink(targetPath) - if err != nil { - return xerrors.Errorf("failed to resolve symlink %s: %w", targetPath, err) - } - - targetStat, err := os.Stat(realTargetPath) + targetStat, err := os.Stat(targetPath) if err != nil { if os.IsNotExist(err) { - return irodsclient_types.NewFileNotFoundError(realTargetPath) + return irodsclient_types.NewFileNotFoundError(targetPath) } - return xerrors.Errorf("failed to stat %s: %w", realTargetPath, err) + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } - // if target is dir, recursive - if targetStat.IsDir() { - // dir - if _, ok := get.inputPathMap[targetPath]; !ok { - // extra dir - logger.Debugf("removing an extra dir %s", targetPath) - removeErr := os.RemoveAll(targetPath) + // target is file + if !targetStat.IsDir() { + if _, ok := get.updatedPathMap[targetPath]; !ok { + // extra file + logger.Debugf("removing an extra file %q", targetPath) + + removeErr := os.Remove(targetPath) now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodGet, - StartAt: now, - EndAt: now, - LocalPath: targetPath, - LocalSize: targetStat.Size(), - Error: removeErr, - Notes: []string{"deleted", "extra"}, + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"extra", "get"}, } get.transferReportManager.AddFile(reportFile) @@ -678,40 +700,26 @@ func (get *GetCommand) deleteExtraInternal(targetPath string) error { if removeErr != nil { return removeErr } - } else { - // non extra dir - entries, err := os.ReadDir(targetPath) - if err != nil { - return xerrors.Errorf("failed to list dir %s: %w", targetPath, err) - } - - for _, entryInDir := range entries { - newTargetPath := commons.MakeTargetLocalFilePath(entryInDir.Name(), targetPath) - err = get.deleteExtraInternal(newTargetPath) - if err != nil { - return err - } - } } return nil } - // file - if _, ok := get.inputPathMap[targetPath]; !ok { - // extra file - logger.Debugf("removing an extra file %s", targetPath) - removeErr := os.Remove(targetPath) + // target is dir + if _, ok := get.updatedPathMap[targetPath]; !ok { + // extra dir + logger.Debugf("removing an extra directory %q", targetPath) + + removeErr := os.RemoveAll(targetPath) now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodGet, - StartAt: now, - EndAt: now, - LocalPath: targetPath, - LocalSize: targetStat.Size(), - Error: removeErr, - Notes: []string{"deleted", "extra"}, + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"extra", "get", "dir"}, } get.transferReportManager.AddFile(reportFile) @@ -719,7 +727,88 @@ func (get *GetCommand) deleteExtraInternal(targetPath string) error { if removeErr != nil { return removeErr } + } else { + // non extra dir + // scan recursively + entries, err := os.ReadDir(targetPath) + if err != nil { + return xerrors.Errorf("failed to list a directory %q: %w", targetPath, err) + } + + for _, entry := range entries { + newTargetPath := path.Join(targetPath, entry.Name()) + err = get.deleteExtraInternal(newTargetPath) + if err != nil { + return err + } + } } return nil } + +func (get *GetCommand) getEncryptionManagerForDecryption(mode commons.EncryptionMode) *commons.EncryptionManager { + manager := commons.NewEncryptionManager(mode) + + switch mode { + case commons.EncryptionModeWinSCP, commons.EncryptionModePGP: + manager.SetKey([]byte(get.decryptionFlagValues.Key)) + case commons.EncryptionModeSSH: + manager.SetPublicPrivateKey(get.decryptionFlagValues.PrivateKeyPath) + } + + return manager +} + +func (get *GetCommand) getPathsForDecryption(sourcePath string, targetPath string) (string, string, error) { + encryptionMode := commons.DetectEncryptionMode(sourcePath) + + if encryptionMode != commons.EncryptionModeUnknown { + // encrypted file + sourceFilename := commons.GetBasename(sourcePath) + encryptManager := get.getEncryptionManagerForDecryption(encryptionMode) + + tempFilePath := commons.MakeTargetLocalFilePath(sourcePath, get.decryptionFlagValues.TempPath) + + decryptedFilename, err := encryptManager.DecryptFilename(sourceFilename) + if err != nil { + return "", "", xerrors.Errorf("failed to decrypt filename %q: %w", sourcePath, err) + } + + targetFilePath := commons.MakeTargetLocalFilePath(decryptedFilename, targetPath) + + return tempFilePath, targetFilePath, nil + } + + targetFilePath := commons.MakeTargetLocalFilePath(sourcePath, targetPath) + + return "", targetFilePath, nil +} + +func (get *GetCommand) decryptFile(sourcePath string, encryptedFilePath string, targetPath string) (bool, error) { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "GetCommand", + "function": "decryptFile", + }) + + encryptionMode := commons.DetectEncryptionMode(sourcePath) + + if encryptionMode != commons.EncryptionModeUnknown { + logger.Debugf("decrypt a data object %q to %q", encryptedFilePath, targetPath) + + encryptManager := get.getEncryptionManagerForDecryption(encryptionMode) + + err := encryptManager.DecryptFile(encryptedFilePath, targetPath) + if err != nil { + return false, xerrors.Errorf("failed to decrypt %q to %q: %w", encryptedFilePath, targetPath, err) + } + + logger.Debugf("removing a temp file %q", encryptedFilePath) + os.Remove(encryptedFilePath) + + return true, nil + } + + return false, nil +} diff --git a/cmd/subcmd/ls.go b/cmd/subcmd/ls.go index b0ff0f5..7420313 100644 --- a/cmd/subcmd/ls.go +++ b/cmd/subcmd/ls.go @@ -111,7 +111,7 @@ func (ls *LsCommand) Process() error { appConfig := commons.GetConfig() syncAccount := false if len(ls.ticketAccessFlagValues.Name) > 0 { - logger.Debugf("use ticket: %s", ls.ticketAccessFlagValues.Name) + logger.Debugf("use ticket %q", ls.ticketAccessFlagValues.Name) appConfig.Ticket = ls.ticketAccessFlagValues.Name syncAccount = true } @@ -138,18 +138,26 @@ func (ls *LsCommand) Process() error { // run for _, sourcePath := range ls.sourcePaths { - err = ls.listOne(sourcePath, false) + err = ls.listOne(sourcePath) if err != nil { - return xerrors.Errorf("failed to perform ls %s: %w", sourcePath, err) + return xerrors.Errorf("failed to list path %q: %w", sourcePath, err) } } return nil } -func (ls *LsCommand) requireDecryptionByMeta(sourceEntry *irodsclient_fs.Entry) bool { - // load encryption config from meta - if !ls.decryptionFlagValues.NoDecryption && !ls.decryptionFlagValues.IgnoreMeta { +func (ls *LsCommand) requireDecryption(sourceEntry *irodsclient_fs.Entry, parentDecryption bool) bool { + if ls.decryptionFlagValues.Decryption { + return true + } + + if ls.decryptionFlagValues.NoDecryption { + return false + } + + if !ls.decryptionFlagValues.IgnoreMeta { + // load encryption config from meta sourceDir := sourceEntry.Path if !sourceEntry.IsDir() { sourceDir = commons.GetDir(sourceEntry.Path) @@ -160,10 +168,10 @@ func (ls *LsCommand) requireDecryptionByMeta(sourceEntry *irodsclient_fs.Entry) return encryptionConfig.Required } - return false + return parentDecryption } -func (ls *LsCommand) listOne(sourcePath string, requireDecryption bool) error { +func (ls *LsCommand) listOne(sourcePath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() @@ -171,41 +179,41 @@ func (ls *LsCommand) listOne(sourcePath string, requireDecryption bool) error { sourceEntry, err := ls.filesystem.Stat(sourcePath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", sourcePath, err) - } + if !irodsclient_types.IsFileNotFoundError(err) { + return xerrors.Errorf("failed to find data-object/collection %q: %w", sourcePath, err) + } - // load encryption config from meta - if !requireDecryption { - requireDecryption = ls.requireDecryptionByMeta(sourceEntry) + return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) } + requireDecryption := ls.requireDecryption(sourceEntry, false) + connection, err := ls.filesystem.GetMetadataConnection() if err != nil { return xerrors.Errorf("failed to get connection: %w", err) } defer ls.filesystem.ReturnMetadataConnection(connection) - collection, err := irodsclient_irodsfs.GetCollection(connection, sourcePath) - if err != nil { - if !irodsclient_types.IsFileNotFoundError(err) { - return xerrors.Errorf("failed to get collection %s: %w", sourcePath, err) + if sourceEntry.IsDir() { + // collection + collection, err := irodsclient_irodsfs.GetCollection(connection, sourcePath) + if err != nil { + return xerrors.Errorf("failed to get collection %q: %w", sourcePath, err) } - } - if err == nil { - // collection colls, err := irodsclient_irodsfs.ListSubCollections(connection, sourcePath) if err != nil { - return xerrors.Errorf("failed to list sub-collections in %s: %w", sourcePath, err) + return xerrors.Errorf("failed to list sub-collections in %q: %w", sourcePath, err) } objs, err := irodsclient_irodsfs.ListDataObjects(connection, collection) if err != nil { - return xerrors.Errorf("failed to list data-objects in %s: %w", sourcePath, err) + return xerrors.Errorf("failed to list data-objects in %q: %w", sourcePath, err) } ls.printDataObjects(objs, requireDecryption) ls.printCollections(colls) + return nil } @@ -214,12 +222,12 @@ func (ls *LsCommand) listOne(sourcePath string, requireDecryption bool) error { parentCollection, err := irodsclient_irodsfs.GetCollection(connection, parentSourcePath) if err != nil { - return xerrors.Errorf("failed to get collection %s: %w", parentSourcePath, err) + return xerrors.Errorf("failed to get collection %q: %w", parentSourcePath, err) } entry, err := irodsclient_irodsfs.GetDataObject(connection, parentCollection, path.Base(sourcePath)) if err != nil { - return xerrors.Errorf("failed to get data-object %s: %w", sourcePath, err) + return xerrors.Errorf("failed to get data-object %q: %w", sourcePath, err) } entries := []*irodsclient_types.IRODSDataObject{entry} @@ -417,7 +425,7 @@ func (ls *LsCommand) printDataObjectShort(entry *irodsclient_types.IRODSDataObje logger.Debugf("%+v", err) newName = fmt.Sprintf("%s\t(decryption_failed)", newName) } else { - newName = fmt.Sprintf("%s\t(encrypted: %s)", newName, decryptedFilename) + newName = fmt.Sprintf("%s\t(encrypted: %q)", newName, decryptedFilename) } } } @@ -464,7 +472,7 @@ func (ls *LsCommand) printReplica(flatReplica FlatReplica, requireDecryption boo logger.Debugf("%+v", err) newName = fmt.Sprintf("%s\tdecryption_failed", newName) } else { - newName = fmt.Sprintf("%s\t(encrypted: %s", newName, decryptedFilename) + newName = fmt.Sprintf("%s\t(encrypted: %q", newName, decryptedFilename) } } } diff --git a/cmd/subcmd/lsmeta.go b/cmd/subcmd/lsmeta.go index 5e9ae4d..e6dc057 100644 --- a/cmd/subcmd/lsmeta.go +++ b/cmd/subcmd/lsmeta.go @@ -6,6 +6,7 @@ import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" "github.com/cyverse/go-irodsclient/irods/types" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" "github.com/spf13/cobra" @@ -32,7 +33,37 @@ func AddLsmetaCommand(rootCmd *cobra.Command) { } func processLsmetaCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + lsMeta, err := NewLsMetaCommand(command, args) + if err != nil { + return err + } + + return lsMeta.Process() +} + +type LsMetaCommand struct { + command *cobra.Command + + listFlagValues *flag.ListFlagValues + targetObjectFlagValues *flag.TargetObjectFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem +} + +func NewLsMetaCommand(command *cobra.Command, args []string) (*LsMetaCommand, error) { + lsMeta := &LsMetaCommand{ + command: command, + + listFlagValues: flag.GetListFlagValues(), + targetObjectFlagValues: flag.GetTargetObjectFlagValues(command), + } + + return lsMeta, nil +} + +func (lsMeta *LsMetaCommand) Process() error { + cont, err := flag.ProcessCommonFlags(lsMeta.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -47,152 +78,84 @@ func processLsmetaCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - listFlagValues := flag.GetListFlagValues() - targetObjectFlagValues := flag.GetTargetObjectFlagValues(command) - // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + lsMeta.account = commons.GetAccount() + lsMeta.filesystem, err = commons.GetIRODSFSClient(lsMeta.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer lsMeta.filesystem.Release() - defer filesystem.Release() - - if targetObjectFlagValues.PathUpdated { - err = listMetaForPath(filesystem, targetObjectFlagValues.Path, listFlagValues) - if err != nil { - return err - } - } else if targetObjectFlagValues.UserUpdated { - err = listMetaForUser(filesystem, targetObjectFlagValues.User, listFlagValues) - if err != nil { - return err - } - } else if targetObjectFlagValues.ResourceUpdated { - err = listMetaForResource(filesystem, targetObjectFlagValues.Resource, listFlagValues) - if err != nil { - return err - } - } else { - // nothing updated - return xerrors.Errorf("path, user, or resource must be given") + if lsMeta.targetObjectFlagValues.PathUpdated { + return lsMeta.listMetaForPath(lsMeta.targetObjectFlagValues.Path) + } else if lsMeta.targetObjectFlagValues.UserUpdated { + return lsMeta.listMetaForUser(lsMeta.targetObjectFlagValues.User) + } else if lsMeta.targetObjectFlagValues.ResourceUpdated { + return lsMeta.listMetaForResource(lsMeta.targetObjectFlagValues.Resource) } - return nil + // nothing updated + return xerrors.Errorf("path, user, or resource must be given") } -func listMetaForPath(fs *irodsclient_fs.FileSystem, targetPath string, listFlagValues *flag.ListFlagValues) error { +func (lsMeta *LsMetaCommand) listMetaForPath(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - metas, err := fs.ListMetadata(targetPath) + metas, err := lsMeta.filesystem.ListMetadata(targetPath) if err != nil { - return xerrors.Errorf("failed to list meta for path %s: %w", targetPath, err) + return xerrors.Errorf("failed to list meta for path %q: %w", targetPath, err) } if len(metas) == 0 { commons.Printf("Found no metadata\n") - } else { - err = printMetas(metas, listFlagValues) - if err != nil { - return err - } + return nil } - return nil + return lsMeta.printMetas(metas) } -func listMetaForUser(fs *irodsclient_fs.FileSystem, username string, listFlagValues *flag.ListFlagValues) error { - metas, err := fs.ListUserMetadata(username) +func (lsMeta *LsMetaCommand) listMetaForUser(username string) error { + metas, err := lsMeta.filesystem.ListUserMetadata(username) if err != nil { - return xerrors.Errorf("failed to list meta for user %s: %w", username, err) + return xerrors.Errorf("failed to list meta for user %q: %w", username, err) } if len(metas) == 0 { commons.Printf("Found no metadata\n") - } else { - err = printMetas(metas, listFlagValues) - if err != nil { - return err - } + return nil } - return nil + return lsMeta.printMetas(metas) } -func listMetaForResource(fs *irodsclient_fs.FileSystem, resource string, listFlagValues *flag.ListFlagValues) error { - metas, err := fs.ListResourceMetadata(resource) +func (lsMeta *LsMetaCommand) listMetaForResource(resource string) error { + metas, err := lsMeta.filesystem.ListResourceMetadata(resource) if err != nil { - return xerrors.Errorf("failed to list meta for resource %s: %w", resource, err) + return xerrors.Errorf("failed to list meta for resource %q: %w", resource, err) } if len(metas) == 0 { commons.Printf("Found no metadata\n") - } else { - err = printMetas(metas, listFlagValues) - if err != nil { - return err - } + return nil } - return nil + return lsMeta.printMetas(metas) } -func getMetaSortFunction(metas []*types.IRODSMeta, sortOrder commons.ListSortOrder, sortReverse bool) func(i int, j int) bool { - if sortReverse { - switch sortOrder { - case commons.ListSortOrderName: - return func(i int, j int) bool { - return metas[i].Name > metas[j].Name - } - case commons.ListSortOrderTime: - return func(i int, j int) bool { - return (metas[i].ModifyTime.After(metas[j].ModifyTime)) || - (metas[i].ModifyTime.Equal(metas[j].ModifyTime) && - metas[i].Name < metas[j].Name) - } - // Cannot sort meta by size or extension, so use default sort by avuid - default: - return func(i int, j int) bool { - return metas[i].AVUID < metas[j].AVUID - } - } - } - - switch sortOrder { - case commons.ListSortOrderName: - return func(i int, j int) bool { - return metas[i].Name < metas[j].Name - } - case commons.ListSortOrderTime: - return func(i int, j int) bool { - return (metas[i].ModifyTime.Before(metas[j].ModifyTime)) || - (metas[i].ModifyTime.Equal(metas[j].ModifyTime) && - metas[i].Name < metas[j].Name) - - } - // Cannot sort meta by size or extension, so use default sort by avuid - default: - return func(i int, j int) bool { - return metas[i].AVUID < metas[j].AVUID - } - } -} - -func printMetas(metas []*types.IRODSMeta, listFlagValues *flag.ListFlagValues) error { - sort.SliceStable(metas, getMetaSortFunction(metas, listFlagValues.SortOrder, listFlagValues.SortReverse)) +func (lsMeta *LsMetaCommand) printMetas(metas []*types.IRODSMeta) error { + sort.SliceStable(metas, lsMeta.getMetaSortFunction(metas, lsMeta.listFlagValues.SortOrder, lsMeta.listFlagValues.SortReverse)) for _, meta := range metas { - printMetaInternal(meta, listFlagValues) + lsMeta.printMetaInternal(meta) } return nil } -func printMetaInternal(meta *types.IRODSMeta, listFlagValues *flag.ListFlagValues) { +func (lsMeta *LsMetaCommand) printMetaInternal(meta *types.IRODSMeta) { createTime := commons.MakeDateTimeString(meta.CreateTime) modTime := commons.MakeDateTimeString(meta.ModifyTime) @@ -217,7 +180,7 @@ func printMetaInternal(meta *types.IRODSMeta, listFlagValues *flag.ListFlagValue units = fmt.Sprintf("\"%s\"", units) } - switch listFlagValues.Format { + switch lsMeta.listFlagValues.Format { case commons.ListFormatLong, commons.ListFormatVeryLong: fmt.Printf("[%s]\n", meta.Name) fmt.Printf("[%s]\n", meta.Name) @@ -233,3 +196,44 @@ func printMetaInternal(meta *types.IRODSMeta, listFlagValues *flag.ListFlagValue fmt.Printf("%d\t%s\t%s\t%s\n", meta.AVUID, name, value, units) } } + +func (lsMeta *LsMetaCommand) getMetaSortFunction(metas []*types.IRODSMeta, sortOrder commons.ListSortOrder, sortReverse bool) func(i int, j int) bool { + if sortReverse { + switch sortOrder { + case commons.ListSortOrderName: + return func(i int, j int) bool { + return metas[i].Name > metas[j].Name + } + case commons.ListSortOrderTime: + return func(i int, j int) bool { + return (metas[i].ModifyTime.After(metas[j].ModifyTime)) || + (metas[i].ModifyTime.Equal(metas[j].ModifyTime) && + metas[i].Name < metas[j].Name) + } + // Cannot sort meta by size or extension, so use default sort by avuid + default: + return func(i int, j int) bool { + return metas[i].AVUID < metas[j].AVUID + } + } + } + + switch sortOrder { + case commons.ListSortOrderName: + return func(i int, j int) bool { + return metas[i].Name < metas[j].Name + } + case commons.ListSortOrderTime: + return func(i int, j int) bool { + return (metas[i].ModifyTime.Before(metas[j].ModifyTime)) || + (metas[i].ModifyTime.Equal(metas[j].ModifyTime) && + metas[i].Name < metas[j].Name) + + } + // Cannot sort meta by size or extension, so use default sort by avuid + default: + return func(i int, j int) bool { + return metas[i].AVUID < metas[j].AVUID + } + } +} diff --git a/cmd/subcmd/lsticket.go b/cmd/subcmd/lsticket.go index 5a1be53..0a4f100 100644 --- a/cmd/subcmd/lsticket.go +++ b/cmd/subcmd/lsticket.go @@ -6,6 +6,7 @@ import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" "github.com/cyverse/go-irodsclient/irods/types" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -32,7 +33,40 @@ func AddLsticketCommand(rootCmd *cobra.Command) { } func processLsticketCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + lsTicket, err := NewLsTicketCommand(command, args) + if err != nil { + return err + } + + return lsTicket.Process() +} + +type LsTicketCommand struct { + command *cobra.Command + + listFlagValues *flag.ListFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + tickets []string +} + +func NewLsTicketCommand(command *cobra.Command, args []string) (*LsTicketCommand, error) { + lsTicket := &LsTicketCommand{ + command: command, + + listFlagValues: flag.GetListFlagValues(), + } + + // tickets + lsTicket.tickets = args + + return lsTicket, nil +} + +func (lsTicket *LsTicketCommand) Process() error { + cont, err := flag.ProcessCommonFlags(lsTicket.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -47,136 +81,73 @@ func processLsticketCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - listFlagValues := flag.GetListFlagValues() - // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + lsTicket.account = commons.GetAccount() + lsTicket.filesystem, err = commons.GetIRODSFSClient(lsTicket.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer lsTicket.filesystem.Release() - defer filesystem.Release() + if len(lsTicket.tickets) == 0 { + return lsTicket.listTickets() + } - if len(args) == 0 { - err = listTicket(filesystem, listFlagValues) + for _, ticketName := range lsTicket.tickets { + err = lsTicket.printTicket(ticketName) if err != nil { - return xerrors.Errorf("failed to perform list ticket: %w", err) - } - } else { - for _, ticketName := range args { - err = getTicket(filesystem, ticketName, listFlagValues) - if err != nil { - return xerrors.Errorf("failed to perform get ticket %s: %w", ticketName, err) - } + return xerrors.Errorf("failed to print ticket %q: %w", ticketName, err) } } return nil } -func getTicketSortFunction(tickets []*types.IRODSTicket, sortOrder commons.ListSortOrder, sortReverse bool) func(i int, j int) bool { - if sortReverse { - switch sortOrder { - case commons.ListSortOrderName: - return func(i int, j int) bool { - return tickets[i].Name > tickets[j].Name - } - case commons.ListSortOrderTime: - return func(i int, j int) bool { - return (tickets[i].ExpirationTime.After(tickets[j].ExpirationTime)) || - (tickets[i].ExpirationTime.Equal(tickets[j].ExpirationTime) && - tickets[i].Name < tickets[j].Name) - } - // Cannot sort tickets by size or extension, so use default sort by name - default: - return func(i int, j int) bool { - return tickets[i].Name < tickets[j].Name - } - } - } - - switch sortOrder { - case commons.ListSortOrderName: - return func(i int, j int) bool { - return tickets[i].Name < tickets[j].Name - } - case commons.ListSortOrderTime: - return func(i int, j int) bool { - return (tickets[i].ExpirationTime.Before(tickets[j].ExpirationTime)) || - (tickets[i].ExpirationTime.Equal(tickets[j].ExpirationTime) && - tickets[i].Name < tickets[j].Name) - - } - // Cannot sort tickets by size or extension, so use default sort by name - default: - return func(i int, j int) bool { - return tickets[i].Name < tickets[j].Name - } - } -} - -func listTicket(fs *irodsclient_fs.FileSystem, listFlagValues *flag.ListFlagValues) error { - tickets, err := fs.ListTickets() +func (lsTicket *LsTicketCommand) listTickets() error { + tickets, err := lsTicket.filesystem.ListTickets() if err != nil { return xerrors.Errorf("failed to list tickets: %w", err) } if len(tickets) == 0 { commons.Printf("Found no tickets\n") - } else { - err = printTickets(fs, tickets, listFlagValues) - if err != nil { - return err - } } - return nil + return lsTicket.printTickets(tickets) } -func getTicket(fs *irodsclient_fs.FileSystem, ticketName string, listFlagValues *flag.ListFlagValues) error { +func (lsTicket *LsTicketCommand) printTicket(ticketName string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "getTicket", + "struct": "LsTicketCommand", + "function": "printTicket", }) - logger.Debugf("get ticket: %s", ticketName) + logger.Debugf("print ticket %q", ticketName) - ticket, err := fs.GetTicket(ticketName) + ticket, err := lsTicket.filesystem.GetTicket(ticketName) if err != nil { - return xerrors.Errorf("failed to get ticket %s: %w", ticketName, err) + return xerrors.Errorf("failed to get ticket %q: %w", ticketName, err) } tickets := []*types.IRODSTicket{ticket} - err = printTickets(fs, tickets, listFlagValues) - if err != nil { - return err - } - - return nil + return lsTicket.printTickets(tickets) } -func printTickets(fs *irodsclient_fs.FileSystem, tickets []*types.IRODSTicket, listFlagValues *flag.ListFlagValues) error { - sort.SliceStable(tickets, getTicketSortFunction(tickets, listFlagValues.SortOrder, listFlagValues.SortReverse)) +func (lsTicket *LsTicketCommand) printTickets(tickets []*types.IRODSTicket) error { + sort.SliceStable(tickets, lsTicket.getTicketSortFunction(tickets, lsTicket.listFlagValues.SortOrder, lsTicket.listFlagValues.SortReverse)) for _, ticket := range tickets { - switch listFlagValues.Format { - case commons.ListFormatLong, commons.ListFormatVeryLong: - restrictions, err := fs.GetTicketRestrictions(ticket.ID) - if err != nil { - return xerrors.Errorf("failed to get ticket restrictions %s: %w", ticket.Name, err) - } - - printTicketInternal(ticket, restrictions) - default: - printTicketInternal(ticket, nil) + err := lsTicket.printTicketInternal(ticket) + if err != nil { + return xerrors.Errorf("failed to print ticket %q: %w", ticket, err) } } return nil } -func printTicketInternal(ticket *types.IRODSTicket, restrictions *irodsclient_fs.IRODSTicketRestrictions) { +func (lsTicket *LsTicketCommand) printTicketInternal(ticket *types.IRODSTicket) error { fmt.Printf("[%s]\n", ticket.Name) fmt.Printf(" id: %d\n", ticket.ID) fmt.Printf(" name: %s\n", ticket.Name) @@ -198,32 +169,82 @@ func printTicketInternal(ticket *types.IRODSTicket, restrictions *irodsclient_fs fmt.Printf(" expiry time: %s\n", commons.MakeDateTimeString(ticket.ExpirationTime)) } - if restrictions != nil { - if len(restrictions.AllowedHosts) == 0 { - fmt.Printf(" No host restrictions\n") - } else { - for _, host := range restrictions.AllowedHosts { - fmt.Printf(" Allowed Hosts:\n") - fmt.Printf(" - %s\n", host) - } + if lsTicket.listFlagValues.Format == commons.ListFormatLong || lsTicket.listFlagValues.Format == commons.ListFormatVeryLong { + restrictions, err := lsTicket.filesystem.GetTicketRestrictions(ticket.ID) + if err != nil { + return xerrors.Errorf("failed to get ticket restrictions %q: %w", ticket.Name, err) } - if len(restrictions.AllowedUserNames) == 0 { - fmt.Printf(" No user restrictions\n") - } else { - for _, user := range restrictions.AllowedUserNames { - fmt.Printf(" Allowed Users:\n") - fmt.Printf(" - %s\n", user) + if restrictions != nil { + if len(restrictions.AllowedHosts) == 0 { + fmt.Printf(" No host restrictions\n") + } else { + for _, host := range restrictions.AllowedHosts { + fmt.Printf(" Allowed Hosts:\n") + fmt.Printf(" - %s\n", host) + } + } + + if len(restrictions.AllowedUserNames) == 0 { + fmt.Printf(" No user restrictions\n") + } else { + for _, user := range restrictions.AllowedUserNames { + fmt.Printf(" Allowed Users:\n") + fmt.Printf(" - %s\n", user) + } + } + + if len(restrictions.AllowedGroupNames) == 0 { + fmt.Printf(" No group restrictions\n") + } else { + for _, group := range restrictions.AllowedGroupNames { + fmt.Printf(" Allowed Groups:\n") + fmt.Printf(" - %s\n", group) + } } } + } - if len(restrictions.AllowedGroupNames) == 0 { - fmt.Printf(" No group restrictions\n") - } else { - for _, group := range restrictions.AllowedGroupNames { - fmt.Printf(" Allowed Groups:\n") - fmt.Printf(" - %s\n", group) + return nil +} + +func (lsTicket *LsTicketCommand) getTicketSortFunction(tickets []*types.IRODSTicket, sortOrder commons.ListSortOrder, sortReverse bool) func(i int, j int) bool { + if sortReverse { + switch sortOrder { + case commons.ListSortOrderName: + return func(i int, j int) bool { + return tickets[i].Name > tickets[j].Name } + case commons.ListSortOrderTime: + return func(i int, j int) bool { + return (tickets[i].ExpirationTime.After(tickets[j].ExpirationTime)) || + (tickets[i].ExpirationTime.Equal(tickets[j].ExpirationTime) && + tickets[i].Name < tickets[j].Name) + } + // Cannot sort tickets by size or extension, so use default sort by name + default: + return func(i int, j int) bool { + return tickets[i].Name < tickets[j].Name + } + } + } + + switch sortOrder { + case commons.ListSortOrderName: + return func(i int, j int) bool { + return tickets[i].Name < tickets[j].Name + } + case commons.ListSortOrderTime: + return func(i int, j int) bool { + return (tickets[i].ExpirationTime.Before(tickets[j].ExpirationTime)) || + (tickets[i].ExpirationTime.Equal(tickets[j].ExpirationTime) && + tickets[i].Name < tickets[j].Name) + + } + // Cannot sort tickets by size or extension, so use default sort by name + default: + return func(i int, j int) bool { + return tickets[i].Name < tickets[j].Name } } } diff --git a/cmd/subcmd/mkdir.go b/cmd/subcmd/mkdir.go index b451710..b07e1f9 100644 --- a/cmd/subcmd/mkdir.go +++ b/cmd/subcmd/mkdir.go @@ -3,6 +3,7 @@ package subcmd import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_irodsfs "github.com/cyverse/go-irodsclient/irods/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -29,7 +30,40 @@ func AddMkdirCommand(rootCmd *cobra.Command) { } func processMkdirCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + mkDir, err := NewMkDirCommand(command, args) + if err != nil { + return err + } + + return mkDir.Process() +} + +type MkDirCommand struct { + command *cobra.Command + + parentsFlagValues *flag.ParentsFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + targetPaths []string +} + +func NewMkDirCommand(command *cobra.Command, args []string) (*MkDirCommand, error) { + mkDir := &MkDirCommand{ + command: command, + + parentsFlagValues: flag.GetParentsFlagValues(), + } + + // target paths + mkDir.targetPaths = args + + return mkDir, nil +} + +func (mkDir *MkDirCommand) Process() error { + cont, err := flag.ProcessCommonFlags(mkDir.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -44,29 +78,28 @@ func processMkdirCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - parentsFlagValues := flag.GetParentsFlagValues() - - // Create a connection - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + // Create a file system + mkDir.account = commons.GetAccount() + mkDir.filesystem, err = commons.GetIRODSFSClient(mkDir.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer mkDir.filesystem.Release() - defer filesystem.Release() - - for _, targetPath := range args { - err = makeOne(filesystem, targetPath, parentsFlagValues) + // run + for _, targetPath := range mkDir.targetPaths { + err = mkDir.makeOne(targetPath) if err != nil { - return xerrors.Errorf("failed to perform mkdir %s: %w", targetPath, err) + return xerrors.Errorf("failed to make a directory %q: %w", targetPath, err) } } return nil } -func makeOne(fs *irodsclient_fs.FileSystem, targetPath string, parentsFlagValues *flag.ParentsFlagValues) error { +func (mkDir *MkDirCommand) makeOne(targetPath string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "MkDirCommand", "function": "makeOne", }) @@ -75,17 +108,18 @@ func makeOne(fs *irodsclient_fs.FileSystem, targetPath string, parentsFlagValues zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - connection, err := fs.GetMetadataConnection() + connection, err := mkDir.filesystem.GetMetadataConnection() if err != nil { return xerrors.Errorf("failed to get connection: %w", err) } - defer fs.ReturnMetadataConnection(connection) + defer mkDir.filesystem.ReturnMetadataConnection(connection) - logger.Debugf("making a collection %s", targetPath) + logger.Debugf("making a collection %q", targetPath) - err = irodsclient_irodsfs.CreateCollection(connection, targetPath, parentsFlagValues.MakeParents) + err = irodsclient_irodsfs.CreateCollection(connection, targetPath, mkDir.parentsFlagValues.MakeParents) if err != nil { - return xerrors.Errorf("failed to create collection %s: %w", targetPath, err) + return xerrors.Errorf("failed to create collection %q: %w", targetPath, err) } + return nil } diff --git a/cmd/subcmd/mkticket.go b/cmd/subcmd/mkticket.go index d21ba72..2b850b6 100644 --- a/cmd/subcmd/mkticket.go +++ b/cmd/subcmd/mkticket.go @@ -3,6 +3,7 @@ package subcmd import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" "github.com/cyverse/go-irodsclient/irods/types" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -29,7 +30,40 @@ func AddMkticketCommand(rootCmd *cobra.Command) { } func processMkticketCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + mkTicket, err := NewMkTicketCommand(command, args) + if err != nil { + return err + } + + return mkTicket.Process() +} + +type MkTicketCommand struct { + command *cobra.Command + + ticketFlagValues *flag.TicketFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + sourcePath string +} + +func NewMkTicketCommand(command *cobra.Command, args []string) (*MkTicketCommand, error) { + mkTicket := &MkTicketCommand{ + command: command, + + ticketFlagValues: flag.GetTicketFlagValues(), + } + + // path + mkTicket.sourcePath = args[0] + + return mkTicket, nil +} + +func (mkTicket *MkTicketCommand) Process() error { + cont, err := flag.ProcessCommonFlags(mkTicket.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -44,40 +78,39 @@ func processMkticketCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - ticketFlagValues := flag.GetTicketFlagValues() - - // Create a connection - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + // Create a file system + mkTicket.account = commons.GetAccount() + mkTicket.filesystem, err = commons.GetIRODSFSClient(mkTicket.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer mkTicket.filesystem.Release() - defer filesystem.Release() - - err = makeTicket(filesystem, ticketFlagValues.Name, ticketFlagValues.Type, args[0]) + // make ticket + err = mkTicket.makeTicket(mkTicket.ticketFlagValues.Name, mkTicket.ticketFlagValues.Type, mkTicket.sourcePath) if err != nil { - return xerrors.Errorf("failed to perform make ticket for %s, %s, %s: %w", ticketFlagValues.Name, ticketFlagValues.Type, args[0], err) + return xerrors.Errorf("failed to make a ticket for %q, %q, %q: %w", mkTicket.ticketFlagValues.Name, mkTicket.ticketFlagValues.Type, mkTicket.sourcePath, err) } return nil } -func makeTicket(fs *irodsclient_fs.FileSystem, ticketName string, ticketType types.TicketType, targetPath string) error { +func (mkTicket *MkTicketCommand) makeTicket(ticketName string, ticketType types.TicketType, targetPath string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "MkTicketCommand", "function": "makeTicket", }) - logger.Debugf("make ticket: %s", ticketName) + logger.Debugf("make ticket %q", ticketName) cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - err := fs.CreateTicket(ticketName, ticketType, targetPath) + err := mkTicket.filesystem.CreateTicket(ticketName, ticketType, targetPath) if err != nil { - return xerrors.Errorf("failed to create ticket %s: %w", ticketName, err) + return xerrors.Errorf("failed to create ticket %q: %w", ticketName, err) } return nil diff --git a/cmd/subcmd/modticket.go b/cmd/subcmd/modticket.go index 5995d93..ba64abe 100644 --- a/cmd/subcmd/modticket.go +++ b/cmd/subcmd/modticket.go @@ -4,6 +4,7 @@ import ( "time" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" "github.com/spf13/cobra" @@ -29,7 +30,39 @@ func AddModticketCommand(rootCmd *cobra.Command) { } func processModticketCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + modTicket, err := NewModTicketCommand(command, args) + if err != nil { + return err + } + + return modTicket.Process() +} + +type ModTicketCommand struct { + command *cobra.Command + + ticketUpdateFlagValues *flag.TicketUpdateFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + tickets []string +} + +func NewModTicketCommand(command *cobra.Command, args []string) (*ModTicketCommand, error) { + modTicket := &ModTicketCommand{ + command: command, + + ticketUpdateFlagValues: flag.GetTicketUpdateFlagValues(command), + } + + modTicket.tickets = args + + return modTicket, nil +} + +func (modTicket *ModTicketCommand) Process() error { + cont, err := flag.ProcessCommonFlags(modTicket.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -44,83 +77,80 @@ func processModticketCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - ticketUpdateFlagValues := flag.GetTicketUpdateFlagValues(command) - - // Create a connection - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + // Create a file system + modTicket.account = commons.GetAccount() + modTicket.filesystem, err = commons.GetIRODSFSClient(modTicket.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer modTicket.filesystem.Release() - defer filesystem.Release() - - for _, ticketName := range args { - if ticketUpdateFlagValues.UseLimitUpdated { - err = modTicketUseLimit(filesystem, ticketName, ticketUpdateFlagValues.UseLimit) + for _, ticketName := range modTicket.tickets { + if modTicket.ticketUpdateFlagValues.UseLimitUpdated { + err = modTicket.modTicketUseLimit(ticketName, modTicket.ticketUpdateFlagValues.UseLimit) if err != nil { return err } } - if ticketUpdateFlagValues.WriteFileLimitUpdated { - err = modTicketWriteFileLimit(filesystem, ticketName, ticketUpdateFlagValues.WriteFileLimit) + if modTicket.ticketUpdateFlagValues.WriteFileLimitUpdated { + err = modTicket.modTicketWriteFileLimit(ticketName, modTicket.ticketUpdateFlagValues.WriteFileLimit) if err != nil { return err } } - if ticketUpdateFlagValues.WriteByteLimitUpdated { - err = modTicketWriteByteLimit(filesystem, ticketName, ticketUpdateFlagValues.WriteByteLimit) + if modTicket.ticketUpdateFlagValues.WriteByteLimitUpdated { + err = modTicket.modTicketWriteByteLimit(ticketName, modTicket.ticketUpdateFlagValues.WriteByteLimit) if err != nil { return err } } - if ticketUpdateFlagValues.ExpirationTimeUpdated { - err = modTicketExpirationTime(filesystem, ticketName, ticketUpdateFlagValues.ExpirationTime) + if modTicket.ticketUpdateFlagValues.ExpirationTimeUpdated { + err = modTicket.modTicketExpirationTime(ticketName, modTicket.ticketUpdateFlagValues.ExpirationTime) if err != nil { return err } } - if len(ticketUpdateFlagValues.AddAllowedUsers) > 0 { - err = modTicketAddAllowedUsers(filesystem, ticketName, ticketUpdateFlagValues.AddAllowedUsers) + if len(modTicket.ticketUpdateFlagValues.AddAllowedUsers) > 0 { + err = modTicket.modTicketAddAllowedUsers(ticketName, modTicket.ticketUpdateFlagValues.AddAllowedUsers) if err != nil { return err } } - if len(ticketUpdateFlagValues.RemoveAllwedUsers) > 0 { - err = modTicketRemoveAllowedUsers(filesystem, ticketName, ticketUpdateFlagValues.RemoveAllwedUsers) + if len(modTicket.ticketUpdateFlagValues.RemoveAllwedUsers) > 0 { + err = modTicket.modTicketRemoveAllowedUsers(ticketName, modTicket.ticketUpdateFlagValues.RemoveAllwedUsers) if err != nil { return err } } - if len(ticketUpdateFlagValues.AddAllowedGroups) > 0 { - err = modTicketAddAllowedGroups(filesystem, ticketName, ticketUpdateFlagValues.AddAllowedGroups) + if len(modTicket.ticketUpdateFlagValues.AddAllowedGroups) > 0 { + err = modTicket.modTicketAddAllowedGroups(ticketName, modTicket.ticketUpdateFlagValues.AddAllowedGroups) if err != nil { return err } } - if len(ticketUpdateFlagValues.RemoveAllowedGroups) > 0 { - err = modTicketRemoveAllowedGroups(filesystem, ticketName, ticketUpdateFlagValues.RemoveAllowedGroups) + if len(modTicket.ticketUpdateFlagValues.RemoveAllowedGroups) > 0 { + err = modTicket.modTicketRemoveAllowedGroups(ticketName, modTicket.ticketUpdateFlagValues.RemoveAllowedGroups) if err != nil { return err } } - if len(ticketUpdateFlagValues.AddAllowedHosts) > 0 { - err = modTicketAddAllowedHosts(filesystem, ticketName, ticketUpdateFlagValues.AddAllowedHosts) + if len(modTicket.ticketUpdateFlagValues.AddAllowedHosts) > 0 { + err = modTicket.modTicketAddAllowedHosts(ticketName, modTicket.ticketUpdateFlagValues.AddAllowedHosts) if err != nil { return err } } - if len(ticketUpdateFlagValues.RemoveAllowedHosts) > 0 { - err = modTicketRemoveAllowedHosts(filesystem, ticketName, ticketUpdateFlagValues.RemoveAllowedHosts) + if len(modTicket.ticketUpdateFlagValues.RemoveAllowedHosts) > 0 { + err = modTicket.modTicketRemoveAllowedHosts(ticketName, modTicket.ticketUpdateFlagValues.RemoveAllowedHosts) if err != nil { return err } @@ -130,93 +160,93 @@ func processModticketCommand(command *cobra.Command, args []string) error { return nil } -func modTicketUseLimit(fs *irodsclient_fs.FileSystem, ticketName string, ulimit int64) error { - err := fs.ModifyTicketUseLimit(ticketName, ulimit) +func (modTicket *ModTicketCommand) modTicketUseLimit(ticketName string, ulimit int64) error { + err := modTicket.filesystem.ModifyTicketUseLimit(ticketName, ulimit) if err != nil { - return xerrors.Errorf("failed to mod ticket (modify uses limit) %s: %w", ticketName, err) + return xerrors.Errorf("failed to mod ticket (modify uses limit) %q: %w", ticketName, err) } return nil } -func modTicketWriteFileLimit(fs *irodsclient_fs.FileSystem, ticketName string, wflimit int64) error { - err := fs.ModifyTicketWriteFileLimit(ticketName, wflimit) +func (modTicket *ModTicketCommand) modTicketWriteFileLimit(ticketName string, wflimit int64) error { + err := modTicket.filesystem.ModifyTicketWriteFileLimit(ticketName, wflimit) if err != nil { - return xerrors.Errorf("failed to mod ticket (modify write file limit) %s: %w", ticketName, err) + return xerrors.Errorf("failed to mod ticket (modify write file limit) %q: %w", ticketName, err) } return nil } -func modTicketWriteByteLimit(fs *irodsclient_fs.FileSystem, ticketName string, wblimit int64) error { - err := fs.ModifyTicketWriteByteLimit(ticketName, wblimit) +func (modTicket *ModTicketCommand) modTicketWriteByteLimit(ticketName string, wblimit int64) error { + err := modTicket.filesystem.ModifyTicketWriteByteLimit(ticketName, wblimit) if err != nil { - return xerrors.Errorf("failed to mod ticket (modify write byte limit) %s: %w", ticketName, err) + return xerrors.Errorf("failed to mod ticket (modify write byte limit) %q: %w", ticketName, err) } return nil } -func modTicketExpirationTime(fs *irodsclient_fs.FileSystem, ticketName string, expiry time.Time) error { - err := fs.ModifyTicketExpirationTime(ticketName, expiry) +func (modTicket *ModTicketCommand) modTicketExpirationTime(ticketName string, expiry time.Time) error { + err := modTicket.filesystem.ModifyTicketExpirationTime(ticketName, expiry) if err != nil { - return xerrors.Errorf("failed to mod ticket (modify expiration time) %s: %w", ticketName, err) + return xerrors.Errorf("failed to mod ticket (modify expiration time) %q: %w", ticketName, err) } return nil } -func modTicketAddAllowedUsers(fs *irodsclient_fs.FileSystem, ticketName string, addUsers []string) error { +func (modTicket *ModTicketCommand) modTicketAddAllowedUsers(ticketName string, addUsers []string) error { for _, addUser := range addUsers { - err := fs.AddTicketAllowedUser(ticketName, addUser) + err := modTicket.filesystem.AddTicketAllowedUser(ticketName, addUser) if err != nil { - return xerrors.Errorf("failed to mod ticket (add allowed user) %s: %w", ticketName, err) + return xerrors.Errorf("failed to mod ticket (add allowed user) %q: %w", ticketName, err) } } return nil } -func modTicketRemoveAllowedUsers(fs *irodsclient_fs.FileSystem, ticketName string, rmUsers []string) error { +func (modTicket *ModTicketCommand) modTicketRemoveAllowedUsers(ticketName string, rmUsers []string) error { for _, rmUser := range rmUsers { - err := fs.RemoveTicketAllowedUser(ticketName, rmUser) + err := modTicket.filesystem.RemoveTicketAllowedUser(ticketName, rmUser) if err != nil { - return xerrors.Errorf("failed to mod ticket (remove allowed user) %s: %w", ticketName, err) + return xerrors.Errorf("failed to mod ticket (remove allowed user) %q: %w", ticketName, err) } } return nil } -func modTicketAddAllowedGroups(fs *irodsclient_fs.FileSystem, ticketName string, addGroups []string) error { +func (modTicket *ModTicketCommand) modTicketAddAllowedGroups(ticketName string, addGroups []string) error { for _, addGroup := range addGroups { - err := fs.AddTicketAllowedUser(ticketName, addGroup) + err := modTicket.filesystem.AddTicketAllowedUser(ticketName, addGroup) if err != nil { - return xerrors.Errorf("failed to mod ticket (add allowed group) %s: %w", ticketName, err) + return xerrors.Errorf("failed to mod ticket (add allowed group) %q: %w", ticketName, err) } } return nil } -func modTicketRemoveAllowedGroups(fs *irodsclient_fs.FileSystem, ticketName string, rmGroups []string) error { +func (modTicket *ModTicketCommand) modTicketRemoveAllowedGroups(ticketName string, rmGroups []string) error { for _, rmGroup := range rmGroups { - err := fs.RemoveTicketAllowedUser(ticketName, rmGroup) + err := modTicket.filesystem.RemoveTicketAllowedUser(ticketName, rmGroup) if err != nil { - return xerrors.Errorf("failed to mod ticket (remove allowed group) %s: %w", ticketName, err) + return xerrors.Errorf("failed to mod ticket (remove allowed group) %q: %w", ticketName, err) } } return nil } -func modTicketAddAllowedHosts(fs *irodsclient_fs.FileSystem, ticketName string, addHosts []string) error { +func (modTicket *ModTicketCommand) modTicketAddAllowedHosts(ticketName string, addHosts []string) error { for _, addHost := range addHosts { - err := fs.AddTicketAllowedHost(ticketName, addHost) + err := modTicket.filesystem.AddTicketAllowedHost(ticketName, addHost) if err != nil { - return xerrors.Errorf("failed to mod ticket (add allowed host) %s: %w", ticketName, err) + return xerrors.Errorf("failed to mod ticket (add allowed host) %q: %w", ticketName, err) } } return nil } -func modTicketRemoveAllowedHosts(fs *irodsclient_fs.FileSystem, ticketName string, rmHosts []string) error { +func (modTicket *ModTicketCommand) modTicketRemoveAllowedHosts(ticketName string, rmHosts []string) error { for _, rmHost := range rmHosts { - err := fs.RemoveTicketAllowedHost(ticketName, rmHost) + err := modTicket.filesystem.RemoveTicketAllowedHost(ticketName, rmHost) if err != nil { - return xerrors.Errorf("failed to mod ticket (remove allowed host) %s: %w", ticketName, err) + return xerrors.Errorf("failed to mod ticket (remove allowed host) %q: %w", ticketName, err) } } return nil diff --git a/cmd/subcmd/mv.go b/cmd/subcmd/mv.go index ab26e8a..c965940 100644 --- a/cmd/subcmd/mv.go +++ b/cmd/subcmd/mv.go @@ -1,7 +1,10 @@ package subcmd import ( + "path" + irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -26,7 +29,38 @@ func AddMvCommand(rootCmd *cobra.Command) { } func processMvCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + mv, err := NewMvCommand(command, args) + if err != nil { + return err + } + + return mv.Process() +} + +type MvCommand struct { + command *cobra.Command + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + sourcePaths []string + targetPath string +} + +func NewMvCommand(command *cobra.Command, args []string) (*MvCommand, error) { + mv := &MvCommand{ + command: command, + } + + // paths + mv.sourcePaths = args[:len(args)-1] + mv.targetPath = args[len(args)-1] + + return mv, nil +} + +func (mv *MvCommand) Process() error { + cont, err := flag.ProcessCommonFlags(mv.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -42,59 +76,154 @@ func processMvCommand(command *cobra.Command, args []string) error { } // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + mv.account = commons.GetAccount() + mv.filesystem, err = commons.GetIRODSFSClient(mv.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer mv.filesystem.Release() - defer filesystem.Release() - - targetPath := args[len(args)-1] - sourcePaths := args[:len(args)-1] + // run + if len(mv.sourcePaths) >= 2 { + // multi-source, target must be a dir + err = mv.ensureTargetIsDir(mv.targetPath) + if err != nil { + return err + } + } - // move - for _, sourcePath := range sourcePaths { - err = moveOne(filesystem, sourcePath, targetPath) + for _, sourcePath := range mv.sourcePaths { + err = mv.moveOne(sourcePath, mv.targetPath) if err != nil { - return xerrors.Errorf("failed to perform mv %s to %s: %w", sourcePath, targetPath, err) + return xerrors.Errorf("failed to move (rename) %q to %q: %w", sourcePath, mv.targetPath, err) } } return nil } -func moveOne(filesystem *irodsclient_fs.FileSystem, sourcePath string, targetPath string) error { - logger := log.WithFields(log.Fields{ - "package": "subcmd", - "function": "moveOne", - }) +func (mv *MvCommand) ensureTargetIsDir(targetPath string) error { + cwd := commons.GetCWD() + home := commons.GetHomeDir() + zone := commons.GetZone() + targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) + + targetEntry, err := mv.filesystem.Stat(targetPath) + if err != nil { + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + + if !targetEntry.IsDir() { + return commons.NewNotDirError(targetPath) + } + return nil +} + +func (mv *MvCommand) moveOne(sourcePath string, targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - sourceEntry, err := filesystem.Stat(sourcePath) + sourceEntry, err := mv.filesystem.Stat(sourcePath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", sourcePath, err) + return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) } - if sourceEntry.Type == irodsclient_fs.FileEntry { - // file - logger.Debugf("renaming a data object %s to %s", sourcePath, targetPath) - err = filesystem.RenameFile(sourcePath, targetPath) - if err != nil { - return xerrors.Errorf("failed to rename %s to %s: %w", sourcePath, targetPath, err) - } - } else { + targetPath = commons.MakeTargetIRODSFilePath(mv.filesystem, sourcePath, targetPath) + + if sourceEntry.IsDir() { // dir - logger.Debugf("renaming a collection %s to %s", sourcePath, targetPath) - err = filesystem.RenameDir(sourcePath, targetPath) - if err != nil { - return xerrors.Errorf("failed to rename %s to %s: %w", sourcePath, targetPath, err) + return mv.moveDir(sourceEntry, targetPath) + } + + // file + return mv.moveFile(sourceEntry, targetPath) +} + +func (mv *MvCommand) moveFile(sourceEntry *irodsclient_fs.Entry, targetPath string) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "MvCommand", + "function": "moveFile", + }) + + targetEntry, err := mv.filesystem.Stat(targetPath) + if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // target does not exist + // target must be a file with new name + logger.Debugf("renaming a data object %q to %q", sourceEntry.Path, targetPath) + err = mv.filesystem.RenameFileToFile(sourceEntry.Path, targetPath) + if err != nil { + return xerrors.Errorf("failed to rename %q to %q: %w", sourceEntry.Path, targetPath, err) + } + + return nil } + + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + + // target exists + // target must be a file + if targetEntry.IsDir() { + return commons.NewNotFileError(targetPath) + } + + // overwrite + err = mv.filesystem.RemoveFile(targetPath, true) + if err != nil { + return xerrors.Errorf("failed to remove %q for overwriting: %w", targetPath, err) + } + + logger.Debugf("renaming a data object %q to %q", sourceEntry.Path, targetPath) + err = mv.filesystem.RenameFileToFile(sourceEntry.Path, targetPath) + if err != nil { + return xerrors.Errorf("failed to rename %q to %q: %w", sourceEntry.Path, targetPath, err) } + return nil } + +func (mv *MvCommand) moveDir(sourceEntry *irodsclient_fs.Entry, targetPath string) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "MvCommand", + "function": "moveDir", + }) + + targetEntry, err := mv.filesystem.Stat(targetPath) + if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // target does not exist + // target must be a directorywith new name + logger.Debugf("renaming a collection %q to %q", sourceEntry.Path, targetPath) + err = mv.filesystem.RenameDirToDir(sourceEntry.Path, targetPath) + if err != nil { + return xerrors.Errorf("failed to rename %q to %q: %w", sourceEntry.Path, targetPath, err) + } + + return nil + } + + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + + // target exist + if targetEntry.IsDir() { + targetDirPath := path.Join(targetPath, sourceEntry.Name) + logger.Debugf("renaming a collection %q to %q", sourceEntry.Path, targetDirPath) + err = mv.filesystem.RenameDirToDir(sourceEntry.Path, targetDirPath) + if err != nil { + return xerrors.Errorf("failed to rename %q to %q: %w", sourceEntry.Path, targetDirPath, err) + } + + return nil + } + + // file + return xerrors.Errorf("failed to rename a collection %q to a file %q: %w", sourceEntry.Path, targetPath) +} diff --git a/cmd/subcmd/passwd.go b/cmd/subcmd/passwd.go index 689dddf..16489be 100644 --- a/cmd/subcmd/passwd.go +++ b/cmd/subcmd/passwd.go @@ -6,6 +6,7 @@ import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_irodsfs "github.com/cyverse/go-irodsclient/irods/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -31,7 +32,31 @@ func AddPasswdCommand(rootCmd *cobra.Command) { } func processPasswdCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + passwd, err := NewPasswdCommand(command, args) + if err != nil { + return err + } + + return passwd.Process() +} + +type PasswdCommand struct { + command *cobra.Command + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem +} + +func NewPasswdCommand(command *cobra.Command, args []string) (*PasswdCommand, error) { + passwd := &PasswdCommand{ + command: command, + } + + return passwd, nil +} + +func (passwd *PasswdCommand) Process() error { + cont, err := flag.ProcessCommonFlags(passwd.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -47,34 +72,33 @@ func processPasswdCommand(command *cobra.Command, args []string) error { } // Create a connection - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + passwd.account = commons.GetAccount() + passwd.filesystem, err = commons.GetIRODSFSClient(passwd.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } - err = changePassword(filesystem) + err = passwd.changePassword() if err != nil { return xerrors.Errorf("failed to change password: %w", err) } return nil } -func changePassword(fs *irodsclient_fs.FileSystem) error { +func (passwd *PasswdCommand) changePassword() error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "PasswdCommand", "function": "changePassword", }) - account := commons.GetAccount() - - connection, err := fs.GetMetadataConnection() + connection, err := passwd.filesystem.GetMetadataConnection() if err != nil { return xerrors.Errorf("failed to get connection: %w", err) } - defer fs.ReturnMetadataConnection(connection) + defer passwd.filesystem.ReturnMetadataConnection(connection) - logger.Debugf("changing password for user %s", account.ClientUser) + logger.Debugf("changing password for user %q", passwd.account.ClientUser) pass := false for i := 0; i < 3; i++ { @@ -87,7 +111,7 @@ func changePassword(fs *irodsclient_fs.FileSystem) error { fmt.Print("\n") currentPassword := string(bytePassword) - if currentPassword == account.Password { + if currentPassword == passwd.account.Password { pass = true break } @@ -112,7 +136,7 @@ func changePassword(fs *irodsclient_fs.FileSystem) error { fmt.Print("\n") newPassword = string(bytePassword) - if newPassword != account.Password { + if newPassword != passwd.account.Password { pass = true break } @@ -139,10 +163,10 @@ func changePassword(fs *irodsclient_fs.FileSystem) error { return xerrors.Errorf("password mismatched") } - err = irodsclient_irodsfs.ChangeUserPassword(connection, account.ClientUser, account.ClientZone, newPassword) + err = irodsclient_irodsfs.ChangeUserPassword(connection, passwd.account.ClientUser, passwd.account.ClientZone, newPassword) if err != nil { - return xerrors.Errorf("failed to change user password for user %s: %w", account.ClientUser, err) + return xerrors.Errorf("failed to change user password for user %q: %w", passwd.account.ClientUser, err) } - return nil + return nil } diff --git a/cmd/subcmd/ps.go b/cmd/subcmd/ps.go index 9c73610..439a797 100644 --- a/cmd/subcmd/ps.go +++ b/cmd/subcmd/ps.go @@ -6,6 +6,7 @@ import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_irodsfs "github.com/cyverse/go-irodsclient/irods/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" "github.com/jedib0t/go-pretty/v6/table" @@ -33,7 +34,35 @@ func AddPsCommand(rootCmd *cobra.Command) { } func processPsCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + ps, err := NewPsCommand(command, args) + if err != nil { + return err + } + + return ps.Process() +} + +type PsCommand struct { + command *cobra.Command + + processFilterFlagValues *flag.ProcessFilterFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem +} + +func NewPsCommand(command *cobra.Command, args []string) (*PsCommand, error) { + ps := &PsCommand{ + command: command, + + processFilterFlagValues: flag.GetProcessFilterFlagValues(), + } + + return ps, nil +} + +func (ps *PsCommand) Process() error { + cont, err := flag.ProcessCommonFlags(ps.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -48,48 +77,46 @@ func processPsCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - processFilterFlagValues := flag.GetProcessFilterFlagValues() - // Create a connection - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + ps.account = commons.GetAccount() + ps.filesystem, err = commons.GetIRODSFSClient(ps.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer ps.filesystem.Release() - defer filesystem.Release() - - err = listProcesses(filesystem, processFilterFlagValues) + err = ps.listProcesses() if err != nil { - return xerrors.Errorf("failed to perform list processes addr %s, zone %s : %w", processFilterFlagValues.Address, processFilterFlagValues.Zone, err) + return xerrors.Errorf("failed to list processes: %w", err) } return nil } -func listProcesses(fs *irodsclient_fs.FileSystem, processFilterFlagValues *flag.ProcessFilterFlagValues) error { +func (ps *PsCommand) listProcesses() error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "PsCommand", "function": "listProcesses", }) - connection, err := fs.GetMetadataConnection() + connection, err := ps.filesystem.GetMetadataConnection() if err != nil { return xerrors.Errorf("failed to get connection: %w", err) } - defer fs.ReturnMetadataConnection(connection) + defer ps.filesystem.ReturnMetadataConnection(connection) - logger.Debugf("listing processes - addr: %s, zone: %s", processFilterFlagValues.Address, processFilterFlagValues.Zone) + logger.Debugf("listing processes - addr: %q, zone: %q", ps.processFilterFlagValues.Address, ps.processFilterFlagValues.Zone) - processes, err := irodsclient_irodsfs.StatProcess(connection, processFilterFlagValues.Address, processFilterFlagValues.Zone) + processes, err := irodsclient_irodsfs.StatProcess(connection, ps.processFilterFlagValues.Address, ps.processFilterFlagValues.Zone) if err != nil { - return xerrors.Errorf("failed to stat process addr %s, zone %s: %w", processFilterFlagValues.Address, processFilterFlagValues.Zone, err) + return xerrors.Errorf("failed to stat process addr %q, zone %q: %w", ps.processFilterFlagValues.Address, ps.processFilterFlagValues.Zone, err) } t := table.NewWriter() t.SetOutputMirror(os.Stdout) - switch processFilterFlagValues.GroupBy { + switch ps.processFilterFlagValues.GroupBy { case flag.ProcessGroupByNone: t.AppendHeader(table.Row{ "Process ID", diff --git a/cmd/subcmd/put.go b/cmd/subcmd/put.go index f1e85a8..dc536c7 100644 --- a/cmd/subcmd/put.go +++ b/cmd/subcmd/put.go @@ -2,9 +2,13 @@ package subcmd import ( "bytes" + "encoding/hex" "fmt" + "io/fs" "os" + "path" "path/filepath" + "time" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" @@ -41,17 +45,95 @@ func AddPutCommand(rootCmd *cobra.Command) { flag.SetSyncFlags(putCmd) flag.SetEncryptionFlags(putCmd) flag.SetPostTransferFlagValues(putCmd) + flag.SetTransferReportFlags(putCmd) rootCmd.AddCommand(putCmd) } func processPutCommand(command *cobra.Command, args []string) error { + put, err := NewPutCommand(command, args) + if err != nil { + return err + } + + return put.Process() +} + +type PutCommand struct { + command *cobra.Command + + forceFlagValues *flag.ForceFlagValues + ticketAccessFlagValues *flag.TicketAccessFlagValues + parallelTransferFlagValues *flag.ParallelTransferFlagValues + progressFlagValues *flag.ProgressFlagValues + retryFlagValues *flag.RetryFlagValues + differentialTransferFlagValues *flag.DifferentialTransferFlagValues + checksumFlagValues *flag.ChecksumFlagValues + noRootFlagValues *flag.NoRootFlagValues + syncFlagValues *flag.SyncFlagValues + encryptionFlagValues *flag.EncryptionFlagValues + postTransferFlagValues *flag.PostTransferFlagValues + transferReportFlagValues *flag.TransferReportFlagValues + + maxConnectionNum int + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + sourcePaths []string + targetPath string + + parallelJobManager *commons.ParallelJobManager + transferReportManager *commons.TransferReportManager + updatedPathMap map[string]bool +} + +func NewPutCommand(command *cobra.Command, args []string) (*PutCommand, error) { + put := &PutCommand{ + command: command, + + forceFlagValues: flag.GetForceFlagValues(), + ticketAccessFlagValues: flag.GetTicketAccessFlagValues(), + parallelTransferFlagValues: flag.GetParallelTransferFlagValues(), + progressFlagValues: flag.GetProgressFlagValues(), + retryFlagValues: flag.GetRetryFlagValues(), + differentialTransferFlagValues: flag.GetDifferentialTransferFlagValues(), + checksumFlagValues: flag.GetChecksumFlagValues(), + noRootFlagValues: flag.GetNoRootFlagValues(), + syncFlagValues: flag.GetSyncFlagValues(), + encryptionFlagValues: flag.GetEncryptionFlagValues(command), + postTransferFlagValues: flag.GetPostTransferFlagValues(), + transferReportFlagValues: flag.GetTransferReportFlagValues(command), + + updatedPathMap: map[string]bool{}, + } + + put.maxConnectionNum = put.parallelTransferFlagValues.ThreadNumber + 2 // 2 for metadata op + + // path + put.targetPath = "./" + put.sourcePaths = args + + if len(args) >= 2 { + put.targetPath = args[len(args)-1] + put.sourcePaths = args[:len(args)-1] + } + + if put.noRootFlagValues.NoRoot && len(put.sourcePaths) > 1 { + return nil, xerrors.Errorf("failed to put multiple source collections without creating root directory") + } + + return put, nil +} + +func (put *PutCommand) Process() error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "processPutCommand", + "struct": "PutCommand", + "function": "Process", }) - cont, err := flag.ProcessCommonFlags(command) + cont, err := flag.ProcessCommonFlags(put.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -66,33 +148,12 @@ func processPutCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - forceFlagValues := flag.GetForceFlagValues() - ticketAccessFlagValues := flag.GetTicketAccessFlagValues() - parallelTransferFlagValues := flag.GetParallelTransferFlagValues() - progressFlagValues := flag.GetProgressFlagValues() - retryFlagValues := flag.GetRetryFlagValues() - differentialTransferFlagValues := flag.GetDifferentialTransferFlagValues() - checksumFlagValues := flag.GetChecksumFlagValues() - noRootFlagValues := flag.GetNoRootFlagValues() - syncFlagValues := flag.GetSyncFlagValues() - encryptionFlagValues := flag.GetEncryptionFlagValues(command) - postTransferFlagValues := flag.GetPostTransferFlagValues() - - maxConnectionNum := parallelTransferFlagValues.ThreadNumber + 2 // 2 for metadata op - - if retryFlagValues.RetryNumber > 0 && !retryFlagValues.RetryChild { - err = commons.RunWithRetry(retryFlagValues.RetryNumber, retryFlagValues.RetryIntervalSeconds) - if err != nil { - return xerrors.Errorf("failed to run with retry %d: %w", retryFlagValues.RetryNumber, err) - } - return nil - } - + // config appConfig := commons.GetConfig() syncAccount := false - if len(ticketAccessFlagValues.Name) > 0 { - logger.Debugf("use ticket: %s", ticketAccessFlagValues.Name) - appConfig.Ticket = ticketAccessFlagValues.Name + if len(put.ticketAccessFlagValues.Name) > 0 { + logger.Debugf("use ticket %q", put.ticketAccessFlagValues.Name) + appConfig.Ticket = put.ticketAccessFlagValues.Name syncAccount = true } @@ -103,60 +164,78 @@ func processPutCommand(command *cobra.Command, args []string) error { } } + // handle retry + if put.retryFlagValues.RetryNumber > 0 && !put.retryFlagValues.RetryChild { + err = commons.RunWithRetry(put.retryFlagValues.RetryNumber, put.retryFlagValues.RetryIntervalSeconds) + if err != nil { + return xerrors.Errorf("failed to run with retry %d: %w", put.retryFlagValues.RetryNumber, err) + } + return nil + } + // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClientAdvanced(account, maxConnectionNum, parallelTransferFlagValues.TCPBufferSize) + put.account = commons.GetAccount() + put.filesystem, err = commons.GetIRODSFSClientAdvanced(put.account, put.maxConnectionNum, put.parallelTransferFlagValues.TCPBufferSize) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer put.filesystem.Release() - defer filesystem.Release() - - // set default key for encryption - if len(encryptionFlagValues.Key) == 0 { - encryptionFlagValues.Key = account.Password - } - - targetPath := "./" - sourcePaths := args[:] - - if len(args) >= 2 { - targetPath = args[len(args)-1] - sourcePaths = args[:len(args)-1] + // transfer report + put.transferReportManager, err = commons.NewTransferReportManager(put.transferReportFlagValues.Report, put.transferReportFlagValues.ReportPath, put.transferReportFlagValues.ReportToStdout) + if err != nil { + return xerrors.Errorf("failed to create transfer report manager: %w", err) } + defer put.transferReportManager.Release() - if noRootFlagValues.NoRoot && len(sourcePaths) > 1 { - return xerrors.Errorf("failed to put multiple source dirs without creating root directory") + // set default key for encryption + if len(put.encryptionFlagValues.Key) == 0 { + put.encryptionFlagValues.Key = put.account.Password } - parallelJobManager := commons.NewParallelJobManager(filesystem, parallelTransferFlagValues.ThreadNumber, progressFlagValues.ShowProgress, progressFlagValues.ShowFullPath) - parallelJobManager.Start() + // parallel job manager + put.parallelJobManager = commons.NewParallelJobManager(put.filesystem, put.parallelTransferFlagValues.ThreadNumber, put.progressFlagValues.ShowProgress, put.progressFlagValues.ShowFullPath) + put.parallelJobManager.Start() - inputPathMap := map[string]bool{} - - for _, sourcePath := range sourcePaths { - newTargetDirPath, err := makePutTargetDirPath(filesystem, sourcePath, targetPath, noRootFlagValues.NoRoot) + // run + if len(put.sourcePaths) >= 2 { + // multi-source, target must be a dir + err = put.ensureTargetIsDir(put.targetPath) if err != nil { - return xerrors.Errorf("failed to make new target path for put %s to %s: %w", sourcePath, targetPath, err) + return err } + } - err = putOne(parallelJobManager, inputPathMap, sourcePath, newTargetDirPath, forceFlagValues, parallelTransferFlagValues, differentialTransferFlagValues, checksumFlagValues, encryptionFlagValues, postTransferFlagValues) + for _, sourcePath := range put.sourcePaths { + err = put.putOne(sourcePath, put.targetPath) if err != nil { - return xerrors.Errorf("failed to perform put %s to %s: %w", sourcePath, targetPath, err) + return xerrors.Errorf("failed to put %q to %q: %w", sourcePath, put.targetPath, err) } } - parallelJobManager.DoneScheduling() - err = parallelJobManager.Wait() + put.parallelJobManager.DoneScheduling() + err = put.parallelJobManager.Wait() if err != nil { return xerrors.Errorf("failed to perform parallel jobs: %w", err) } + // delete on success + if put.postTransferFlagValues.DeleteOnSuccess { + for _, sourcePath := range put.sourcePaths { + logger.Infof("deleting source %q after successful data put", sourcePath) + + err := put.deleteOnSuccess(sourcePath) + if err != nil { + return xerrors.Errorf("failed to delete source %q: %w", sourcePath, err) + } + } + } + // delete extra - if syncFlagValues.Delete { - logger.Infof("deleting extra files and dirs under %s", targetPath) + if put.syncFlagValues.Delete { + logger.Infof("deleting extra files and directories under %q", put.targetPath) - err = putDeleteExtra(filesystem, inputPathMap, targetPath) + err = put.deleteExtra(put.targetPath) if err != nil { return xerrors.Errorf("failed to delete extra files: %w", err) } @@ -165,401 +244,589 @@ func processPutCommand(command *cobra.Command, args []string) error { return nil } -func getEncryptionManagerForEncrypt(encryptionFlagValues *flag.EncryptionFlagValues) *commons.EncryptionManager { - manager := commons.NewEncryptionManager(encryptionFlagValues.Mode) - - switch encryptionFlagValues.Mode { - case commons.EncryptionModeWinSCP, commons.EncryptionModePGP: - manager.SetKey([]byte(encryptionFlagValues.Key)) - case commons.EncryptionModeSSH: - manager.SetPublicPrivateKey(encryptionFlagValues.PublicPrivateKeyPath) - } - - return manager -} - -func putOne(parallelJobManager *commons.ParallelJobManager, inputPathMap map[string]bool, sourcePath string, targetPath string, forceFlagValues *flag.ForceFlagValues, parallelTransferFlagValues *flag.ParallelTransferFlagValues, differentialTransferFlagValues *flag.DifferentialTransferFlagValues, checksumFlagValues *flag.ChecksumFlagValues, encryptionFlagValues *flag.EncryptionFlagValues, postTransferFlagValues *flag.PostTransferFlagValues) error { - logger := log.WithFields(log.Fields{ - "package": "subcmd", - "function": "putOne", - }) - +func (put *PutCommand) ensureTargetIsDir(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() - sourcePath = commons.MakeLocalPath(sourcePath) targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - filesystem := parallelJobManager.GetFilesystem() - - realSourcePath, err := commons.ResolveSymlink(sourcePath) + targetEntry, err := put.filesystem.Stat(targetPath) if err != nil { - return xerrors.Errorf("failed to resolve symlink %s: %w", sourcePath, err) + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } - logger.Debugf("path %s ==> %s", sourcePath, realSourcePath) + if !targetEntry.IsDir() { + return commons.NewNotDirError(targetPath) + } - sourceStat, err := os.Stat(realSourcePath) - if err != nil { - if os.IsNotExist(err) { - return irodsclient_types.NewFileNotFoundError(realSourcePath) - } + return nil +} + +func (put *PutCommand) requireEncryption(targetPath string, parentEncryption bool, parentEncryptionMode commons.EncryptionMode) (bool, commons.EncryptionMode) { + if put.encryptionFlagValues.Encryption { + return true, put.encryptionFlagValues.Mode + } - return xerrors.Errorf("failed to stat %s: %w", realSourcePath, err) + if put.encryptionFlagValues.NoEncryption { + return false, commons.EncryptionModeUnknown } - // load encryption config from meta - if !encryptionFlagValues.NoEncryption && !encryptionFlagValues.IgnoreMeta { + if !put.encryptionFlagValues.IgnoreMeta { + // load encryption config from meta targetDir := targetPath - targetEntry, err := filesystem.Stat(targetPath) + + targetEntry, err := put.filesystem.Stat(targetPath) if err != nil { if irodsclient_types.IsFileNotFoundError(err) { - // target path is file name - targetDir = commons.GetDir(targetPath) + targetDir = commons.GetDir(targetEntry.Path) } else { - return xerrors.Errorf("failed to stat %s: %w", targetPath, err) + return parentEncryption, parentEncryptionMode } } else { if !targetEntry.IsDir() { - targetDir = commons.GetDir(targetPath) + targetDir = commons.GetDir(targetEntry.Path) } } - encryptionConfig := commons.GetEncryptionConfigFromMeta(filesystem, targetDir) - if encryptionConfig.Required { - encryptionFlagValues.Encryption = encryptionConfig.Required - if encryptionConfig.Mode != commons.EncryptionModeUnknown { - encryptionFlagValues.Mode = encryptionConfig.Mode + encryptionConfig := commons.GetEncryptionConfigFromMeta(put.filesystem, targetDir) + + if encryptionConfig.Mode == commons.EncryptionModeUnknown { + if put.encryptionFlagValues.Mode == commons.EncryptionModeUnknown { + return false, commons.EncryptionModeUnknown } + + return encryptionConfig.Required, put.encryptionFlagValues.Mode } - } - originalSourcePath := sourcePath + return encryptionConfig.Required, encryptionConfig.Mode + } - if !sourceStat.IsDir() { - // file - // encrypt first if necessary - if encryptionFlagValues.Encryption { - logger.Debugf("encrypting file %s", sourcePath) + return parentEncryption, parentEncryptionMode +} - encryptManager := getEncryptionManagerForEncrypt(encryptionFlagValues) - newFilename, err := encryptManager.EncryptFilename(sourceStat.Name()) - if err != nil { - return xerrors.Errorf("failed to encrypt %s: %w", sourcePath, err) - } +func (put *PutCommand) putOne(sourcePath string, targetPath string) error { + cwd := commons.GetCWD() + home := commons.GetHomeDir() + zone := commons.GetZone() + sourcePath = commons.MakeLocalPath(sourcePath) + targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - encryptedSourcePath := filepath.Join(encryptionFlagValues.TempPath, newFilename) - err = encryptManager.EncryptFile(sourcePath, encryptedSourcePath) - if err != nil { - return xerrors.Errorf("failed to encrypt %s: %w", sourcePath, err) - } + sourceStat, err := os.Stat(sourcePath) + if err != nil { + if os.IsNotExist(err) { + return irodsclient_types.NewFileNotFoundError(sourcePath) + } - encryptedSourceStat, err := os.Stat(encryptedSourcePath) - if err != nil { - return xerrors.Errorf("failed to stat file %s: %w", encryptedSourcePath, err) - } + return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) + } - sourcePath = encryptedSourcePath - sourceStat = encryptedSourceStat + if sourceStat.IsDir() { + // dir + if !put.noRootFlagValues.NoRoot { + targetPath = commons.MakeTargetIRODSFilePath(put.filesystem, sourcePath, targetPath) } - targetFilePath := commons.MakeTargetIRODSFilePath(filesystem, sourcePath, targetPath) - commons.MarkPathMap(inputPathMap, targetFilePath) + return put.putDir(sourceStat, sourcePath, targetPath, false, commons.EncryptionModeUnknown) + } - fileExist := false - targetEntry, err := filesystem.StatFile(targetFilePath) + // file + requireEncryption, encryptionMode := put.requireEncryption(targetPath, false, commons.EncryptionModeUnknown) + if requireEncryption { + // encrypt filename + tempPath, newTargetPath, err := put.getPathsForEncryption(sourcePath, targetPath) if err != nil { - if !irodsclient_types.IsFileNotFoundError(err) { - return xerrors.Errorf("failed to stat %s: %w", targetFilePath, err) - } - } else { - fileExist = true + return xerrors.Errorf("failed to get encryption path for %q: %w", sourcePath, err) } - putTask := func(job *commons.ParallelJob) error { - manager := job.GetManager() - fs := manager.GetFilesystem() + return put.putFile(sourceStat, sourcePath, tempPath, newTargetPath, requireEncryption, encryptionMode) + } - callbackPut := func(processed int64, total int64) { - job.Progress(processed, total, false) - } + targetPath = commons.MakeTargetIRODSFilePath(put.filesystem, sourcePath, targetPath) + return put.putFile(sourceStat, sourcePath, "", targetPath, requireEncryption, commons.EncryptionModeUnknown) +} - job.Progress(0, sourceStat.Size(), false) +func (put *PutCommand) schedulePut(sourceStat fs.FileInfo, sourcePath string, tempPath string, targetPath string, requireDecryption bool, encryptionMode commons.EncryptionMode) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "PutCommand", + "function": "schedulePut", + }) - logger.Debugf("uploading a file %s to %s", sourcePath, targetFilePath) + putTask := func(job *commons.ParallelJob) error { + manager := job.GetManager() + fs := manager.GetFilesystem() - var uploadErr error + callbackPut := func(processed int64, total int64) { + job.Progress(processed, total, false) + } - // determine how to download + job.Progress(0, sourceStat.Size(), false) - if parallelTransferFlagValues.SingleTread || parallelTransferFlagValues.ThreadNumber == 1 { - _, uploadErr = fs.UploadFile(sourcePath, targetFilePath, "", false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) - } else if parallelTransferFlagValues.RedirectToResource { - _, uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) - } else if parallelTransferFlagValues.Icat { - _, uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) - } else { - // auto - if sourceStat.Size() >= commons.RedirectToResourceMinSize { - // redirect-to-resource - _, uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) - } else { - if filesystem.SupportParallelUpload() { - _, uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) - } else { - if sourceStat.Size() >= commons.ParallelUploadMinSize { - // does not support parall upload via iCAT - // redirect-to-resource - _, uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) - } else { - _, uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, checksumFlagValues.CalculateChecksum, checksumFlagValues.VerifyChecksum, callbackPut) - } - } - } - } - - if uploadErr != nil { - job.Progress(-1, sourceStat.Size(), true) - return xerrors.Errorf("failed to upload %s to %s: %w", sourcePath, targetFilePath, uploadErr) - } + logger.Debugf("uploading a file %q to %q", sourcePath, targetPath) - logger.Debugf("uploaded a file %s to %s", sourcePath, targetFilePath) - job.Progress(sourceStat.Size(), sourceStat.Size(), false) + var uploadErr error + var uploadResult *irodsclient_fs.FileTransferResult + notes := []string{} - if encryptionFlagValues.Encryption { - logger.Debugf("removing a temp file %s", sourcePath) - os.Remove(sourcePath) + // encrypt + if requireDecryption { + encrypted, err := put.encryptFile(sourcePath, tempPath, targetPath, encryptionMode) + if err != nil { + job.Progress(-1, sourceStat.Size(), true) + return xerrors.Errorf("failed to decrypt file: %w", err) } - if postTransferFlagValues.DeleteOnSuccess { - logger.Debugf("removing source file %s", originalSourcePath) - os.Remove(originalSourcePath) + if encrypted { + notes = append(notes, "encrypted", targetPath) } + } - return nil + uploadSourcePath := sourcePath + if len(tempPath) > 0 { + uploadSourcePath = tempPath } - if fileExist { - if differentialTransferFlagValues.DifferentialTransfer { - if differentialTransferFlagValues.NoHash { - if targetEntry.Size == sourceStat.Size() { - commons.Printf("skip uploading a file %s. The file already exists!\n", targetFilePath) - logger.Debugf("skip uploading a file %s. The file already exists!", targetFilePath) - return nil - } - } else { - if targetEntry.Size == sourceStat.Size() { - if len(targetEntry.CheckSum) > 0 { - // compare hash - hash, err := irodsclient_util.HashLocalFile(sourcePath, string(targetEntry.CheckSumAlgorithm)) - if err != nil { - return xerrors.Errorf("failed to get hash for %s: %w", sourcePath, err) - } - - if bytes.Equal(hash, targetEntry.CheckSum) { - commons.Printf("skip uploading a file %s. The file with the same hash already exists!\n", targetFilePath) - logger.Debugf("skip uploading a file %s. The file with the same hash already exists!", targetFilePath) - return nil - } - } - } - } + // determine how to upload + if put.parallelTransferFlagValues.SingleTread || put.parallelTransferFlagValues.ThreadNumber == 1 { + uploadResult, uploadErr = fs.UploadFile(uploadSourcePath, targetPath, "", false, put.checksumFlagValues.CalculateChecksum, put.checksumFlagValues.VerifyChecksum, callbackPut) + notes = append(notes, "icat", "single-thread") + } else if put.parallelTransferFlagValues.RedirectToResource { + uploadResult, uploadErr = fs.UploadFileParallelRedirectToResource(uploadSourcePath, targetPath, "", 0, false, put.checksumFlagValues.CalculateChecksum, put.checksumFlagValues.VerifyChecksum, callbackPut) + notes = append(notes, "redirect-to-resource") + } else if put.parallelTransferFlagValues.Icat { + uploadResult, uploadErr = fs.UploadFileParallel(uploadSourcePath, targetPath, "", 0, false, put.checksumFlagValues.CalculateChecksum, put.checksumFlagValues.VerifyChecksum, callbackPut) + notes = append(notes, "icat", "multi-thread") + } else { + // auto + if sourceStat.Size() >= commons.RedirectToResourceMinSize { + // redirect-to-resource + uploadResult, uploadErr = fs.UploadFileParallelRedirectToResource(uploadSourcePath, targetPath, "", 0, false, put.checksumFlagValues.CalculateChecksum, put.checksumFlagValues.VerifyChecksum, callbackPut) + notes = append(notes, "redirect-to-resource") } else { - if !forceFlagValues.Force { - // ask - overwrite := commons.InputYN(fmt.Sprintf("file %s already exists. Overwrite?", targetFilePath)) - if !overwrite { - commons.Printf("skip uploading a file %s. The data object already exists!\n", targetFilePath) - logger.Debugf("skip uploading a file %s. The data object already exists!", targetFilePath) - return nil - } - } + uploadResult, uploadErr = fs.UploadFileParallel(uploadSourcePath, targetPath, "", 0, false, put.checksumFlagValues.CalculateChecksum, put.checksumFlagValues.VerifyChecksum, callbackPut) + notes = append(notes, "icat", "multi-thread") } } - threadsRequired := computeThreadsRequiredForPut(filesystem, parallelTransferFlagValues.SingleTread, sourceStat.Size()) - parallelJobManager.Schedule(sourcePath, putTask, threadsRequired, progress.UnitsBytes) - logger.Debugf("scheduled a local file upload %s to %s", sourcePath, targetFilePath) - } else { - logger.Debugf("uploading a local directory %s to %s", sourcePath, targetPath) + if uploadErr != nil { + job.Progress(-1, sourceStat.Size(), true) + return xerrors.Errorf("failed to upload %q to %q: %w", sourcePath, targetPath, uploadErr) + } - entries, err := os.ReadDir(sourcePath) + err := put.transferReportManager.AddTransfer(uploadResult, commons.TransferMethodPut, uploadErr, notes) if err != nil { - return xerrors.Errorf("failed to read dir %s: %w", sourcePath, err) + job.Progress(-1, sourceStat.Size(), true) + return xerrors.Errorf("failed to add transfer report: %w", err) } - for _, entry := range entries { - encryptionFlagValuesCopy := encryptionFlagValues + if requireDecryption { + logger.Debugf("removing a temp file %q", tempPath) + os.Remove(tempPath) + } - newSourcePath := filepath.Join(sourcePath, entry.Name()) + logger.Debugf("uploaded a file %q to %q", sourcePath, targetPath) + job.Progress(sourceStat.Size(), sourceStat.Size(), false) - realNewSourcePath, err := commons.ResolveSymlink(newSourcePath) - if err != nil { - return xerrors.Errorf("failed to resolve symlink %s: %w", newSourcePath, err) - } + return nil + } - newSourceStat, err := os.Stat(realNewSourcePath) - if err != nil { - if os.IsNotExist(err) { - return irodsclient_types.NewFileNotFoundError(realNewSourcePath) + threadsRequired := put.computeThreadsRequired(sourceStat.Size()) + err := put.parallelJobManager.Schedule(sourcePath, putTask, threadsRequired, progress.UnitsBytes) + if err != nil { + return xerrors.Errorf("failed to schedule upload %q to %q: %w", sourcePath, targetPath, err) + } + + commons.MarkPathMap(put.updatedPathMap, targetPath) + + logger.Debugf("scheduled a file upload %q to %q", sourcePath, targetPath) + + return nil +} + +func (put *PutCommand) putFile(sourceStat fs.FileInfo, sourcePath string, tempPath string, targetPath string, requireEncryption bool, encryptionMode commons.EncryptionMode) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "PutCommand", + "function": "putFile", + }) + + targetEntry, err := put.filesystem.Stat(targetPath) + if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // target does not exist + // target must be a file with new name + return put.schedulePut(sourceStat, sourcePath, tempPath, targetPath, requireEncryption, encryptionMode) + } + + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + + // target exists + // target must be a file + if targetEntry.IsDir() { + return commons.NewNotFileError(targetPath) + } + + if put.differentialTransferFlagValues.DifferentialTransfer { + if put.differentialTransferFlagValues.NoHash { + if targetEntry.Size == sourceStat.Size() { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + SourceSize: sourceStat.Size(), + + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"differential", "no_hash", "same file size", "skip"}, } - return xerrors.Errorf("failed to stat %s: %w", realNewSourcePath, err) + put.transferReportManager.AddFile(reportFile) + + commons.Printf("skip uploading a file %q to %q. The file already exists!\n", sourcePath, targetPath) + logger.Debugf("skip uploading a file %q to %q. The file already exists!", sourcePath, targetPath) + return nil } + } else { + if targetEntry.Size == sourceStat.Size() { + // compare hash + if len(targetEntry.CheckSum) > 0 { + localChecksum, err := irodsclient_util.HashLocalFile(sourcePath, string(targetEntry.CheckSumAlgorithm)) + if err != nil { + return xerrors.Errorf("failed to get hash for %q: %w", sourcePath, err) + } + + if bytes.Equal(localChecksum, targetEntry.CheckSum) { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + SourceSize: sourceStat.Size(), + SourceChecksum: hex.EncodeToString(localChecksum), + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"differential", "same checksum", "skip"}, + } - targetDirPath := targetPath - if newSourceStat.IsDir() { - // dir - targetDirPath = commons.MakeTargetIRODSFilePath(filesystem, entry.Name(), targetPath) + put.transferReportManager.AddFile(reportFile) - if !filesystem.ExistsDir(targetDirPath) { - // not exist - err = filesystem.MakeDir(targetDirPath, true) - if err != nil { - return xerrors.Errorf("failed to make dir %s: %w", targetDirPath, err) + commons.Printf("skip uploading a file %q to %q. The file with the same hash already exists!\n", sourcePath, targetPath) + logger.Debugf("skip uploading a file %q to %q. The file with the same hash already exists!", sourcePath, targetPath) + return nil } } } + } + } else { + if !put.forceFlagValues.Force { + // ask + overwrite := commons.InputYN(fmt.Sprintf("file %q already exists. Overwrite?", targetPath)) + if !overwrite { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + SourceSize: sourceStat.Size(), + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"no_overwrite", "skip"}, + } - commons.MarkPathMap(inputPathMap, targetDirPath) + put.transferReportManager.AddFile(reportFile) - err = putOne(parallelJobManager, inputPathMap, newSourcePath, targetDirPath, forceFlagValues, parallelTransferFlagValues, differentialTransferFlagValues, checksumFlagValues, encryptionFlagValuesCopy, postTransferFlagValues) - if err != nil { - return xerrors.Errorf("failed to perform put %s to %s: %w", newSourcePath, targetDirPath, err) + commons.Printf("skip uploading a file %q to %q. The data object already exists!\n", sourcePath, targetPath) + logger.Debugf("skip uploading a file %q to %q. The data object already exists!", sourcePath, targetPath) + return nil } } } - return nil + + // schedule + return put.schedulePut(sourceStat, sourcePath, tempPath, targetPath, requireEncryption, encryptionMode) } -func makePutTargetDirPath(filesystem *irodsclient_fs.FileSystem, sourcePath string, targetPath string, noRoot bool) (string, error) { - logger := log.WithFields(log.Fields{ - "package": "subcmd", - "function": "makePutTargetDirPath", - }) +func (put *PutCommand) putDir(sourceStat fs.FileInfo, sourcePath string, targetPath string, parentEncryption bool, parentEncryptionMode commons.EncryptionMode) error { + targetEntry, err := put.filesystem.Stat(targetPath) + if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // target does not exist + // target must be a directorywith new name + err = put.filesystem.MakeDir(targetPath, true) + if err != nil { + return xerrors.Errorf("failed to make a collection %q: %w", targetPath, err) + } - cwd := commons.GetCWD() - home := commons.GetHomeDir() - zone := commons.GetZone() - sourcePath = commons.MakeLocalPath(sourcePath) - targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + DestPath: targetPath, + Notes: []string{"directory"}, + } - realSourcePath, err := commons.ResolveSymlink(sourcePath) - if err != nil { - return "", xerrors.Errorf("failed to resolve symlink %s: %w", sourcePath, err) + put.transferReportManager.AddFile(reportFile) + } else { + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + } else { + // target exists + if !targetEntry.IsDir() { + return commons.NewNotDirError(targetPath) + } } - logger.Debugf("path %s ==> %s", sourcePath, realSourcePath) + requireEncryption, encryptionMode := put.requireEncryption(targetPath, parentEncryption, parentEncryptionMode) - sourceStat, err := os.Stat(realSourcePath) + // get entries + entries, err := os.ReadDir(sourcePath) if err != nil { - if os.IsNotExist(err) { - return "", irodsclient_types.NewFileNotFoundError(realSourcePath) - } - - return "", xerrors.Errorf("failed to stat %s: %w", realSourcePath, err) + return xerrors.Errorf("failed to list a directory %q: %w", sourcePath, err) } - if !sourceStat.IsDir() { - // file - targetFilePath := commons.MakeTargetIRODSFilePath(filesystem, sourcePath, targetPath) - targetDirPath := commons.GetDir(targetFilePath) - _, err := filesystem.Stat(targetDirPath) - if err != nil { - return "", xerrors.Errorf("failed to stat dir %s: %w", targetDirPath, err) - } + for _, entry := range entries { + newEntryPath := commons.MakeTargetIRODSFilePath(put.filesystem, entry.Name(), targetPath) - return targetDirPath, nil - } else { - // dir - _, err := filesystem.Stat(targetPath) + entryPath := filepath.Join(sourcePath, entry.Name()) + + entryStat, err := os.Stat(entryPath) if err != nil { - return "", xerrors.Errorf("failed to stat dir %s: %w", targetPath, err) - } + if os.IsNotExist(err) { + return irodsclient_types.NewFileNotFoundError(entryPath) + } - targetDirPath := targetPath + return xerrors.Errorf("failed to stat %q: %w", entryPath, err) + } - if !noRoot { - // make target dir - targetDirPath = commons.MakeTargetIRODSFilePath(filesystem, sourcePath, targetPath) - err = filesystem.MakeDir(targetDirPath, true) + if entryStat.IsDir() { + // dir + err = put.putDir(entryStat, entryPath, newEntryPath, requireEncryption, encryptionMode) if err != nil { - return "", xerrors.Errorf("failed to make dir %s: %w", targetDirPath, err) + return err } - } - return targetDirPath, nil + commons.MarkPathMap(put.updatedPathMap, newEntryPath) + } else { + // file + if requireEncryption { + // encrypt filename + tempPath, newTargetPath, err := put.getPathsForEncryption(entryPath, targetPath) + if err != nil { + return xerrors.Errorf("failed to get encryption path for %q: %w", entryPath, err) + } + + err = put.putFile(entryStat, entryPath, tempPath, newTargetPath, requireEncryption, encryptionMode) + if err != nil { + return err + } + + commons.MarkPathMap(put.updatedPathMap, newTargetPath) + } else { + err = put.putFile(entryStat, entryPath, "", newEntryPath, requireEncryption, encryptionMode) + if err != nil { + return err + } + + commons.MarkPathMap(put.updatedPathMap, newEntryPath) + } + } } + + commons.MarkPathMap(put.updatedPathMap, targetPath) + + return nil } -func computeThreadsRequiredForPut(fs *irodsclient_fs.FileSystem, singleThreaded bool, size int64) int { - if singleThreaded { +func (put *PutCommand) computeThreadsRequired(size int64) int { + if put.parallelTransferFlagValues.SingleTread { return 1 } - if fs.SupportParallelUpload() { - return irodsclient_util.GetNumTasksForParallelTransfer(size) + if put.filesystem.SupportParallelUpload() { + numTasks := irodsclient_util.GetNumTasksForParallelTransfer(size) + if put.parallelTransferFlagValues.ThreadNumber < numTasks { + return put.parallelTransferFlagValues.ThreadNumber + } + + return numTasks } return 1 } -func putDeleteExtra(filesystem *irodsclient_fs.FileSystem, inputPathMap map[string]bool, targetPath string) error { +func (put *PutCommand) deleteOnSuccess(sourcePath string) error { + sourceStat, err := os.Stat(sourcePath) + if err != nil { + return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) + } + + if sourceStat.IsDir() { + return os.RemoveAll(sourcePath) + } + + return os.Remove(sourcePath) +} + +func (put *PutCommand) deleteExtra(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - return putDeleteExtraInternal(filesystem, inputPathMap, targetPath) + return put.deleteExtraInternal(targetPath) } -func putDeleteExtraInternal(filesystem *irodsclient_fs.FileSystem, inputPathMap map[string]bool, targetPath string) error { +func (put *PutCommand) deleteExtraInternal(targetPath string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "putDeleteExtraInternal", + "struct": "PutCommand", + "function": "deleteExtraInternal", }) - targetEntry, err := filesystem.Stat(targetPath) + targetEntry, err := put.filesystem.Stat(targetPath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", targetPath, err) + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } - if targetEntry.Type == irodsclient_fs.FileEntry { + if !targetEntry.IsDir() { // file - if _, ok := inputPathMap[targetPath]; !ok { + if _, ok := put.updatedPathMap[targetPath]; !ok { // extra file - logger.Debugf("removing an extra data object %s", targetPath) - removeErr := filesystem.RemoveFile(targetPath, true) + logger.Debugf("removing an extra data object %q", targetPath) + + removeErr := put.filesystem.RemoveFile(targetPath, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"extra", "put"}, + } + + put.transferReportManager.AddFile(reportFile) + if removeErr != nil { return removeErr } } + + return nil + } + + // target is dir + if _, ok := put.updatedPathMap[targetPath]; !ok { + // extra dir + logger.Debugf("removing an extra collection %q", targetPath) + + removeErr := put.filesystem.RemoveDir(targetPath, true, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"extra", "put", "dir"}, + } + + put.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } } else { - // dir - if _, ok := inputPathMap[targetPath]; !ok { - // extra dir - logger.Debugf("removing an extra collection %s", targetPath) - removeErr := filesystem.RemoveDir(targetPath, true, true) - if removeErr != nil { - return removeErr - } - } else { - // non extra dir - entries, err := filesystem.List(targetPath) + // non extra dir + // scan recursively + entries, err := put.filesystem.List(targetPath) + if err != nil { + return xerrors.Errorf("failed to list a collection %q: %w", targetPath, err) + } + + for _, entry := range entries { + newTargetPath := path.Join(targetPath, entry.Name) + err = put.deleteExtraInternal(newTargetPath) if err != nil { - return xerrors.Errorf("failed to list dir %s: %w", targetPath, err) + return err } + } + } - for idx := range entries { - newTargetPath := entries[idx].Path + return nil +} - err = putDeleteExtraInternal(filesystem, inputPathMap, newTargetPath) - if err != nil { - return err - } - } +func (put *PutCommand) getEncryptionManagerForEncryption(mode commons.EncryptionMode) *commons.EncryptionManager { + manager := commons.NewEncryptionManager(mode) + + switch mode { + case commons.EncryptionModeWinSCP, commons.EncryptionModePGP: + manager.SetKey([]byte(put.encryptionFlagValues.Key)) + case commons.EncryptionModeSSH: + manager.SetPublicPrivateKey(put.encryptionFlagValues.PublicPrivateKeyPath) + } + + return manager +} + +func (put *PutCommand) getPathsForEncryption(sourcePath string, targetPath string) (string, string, error) { + if put.encryptionFlagValues.Mode != commons.EncryptionModeUnknown { + encryptManager := put.getEncryptionManagerForEncryption(put.encryptionFlagValues.Mode) + sourceFilename := commons.GetBasename(sourcePath) + + encryptedFilename, err := encryptManager.EncryptFilename(sourceFilename) + if err != nil { + return "", "", xerrors.Errorf("failed to encrypt filename %q: %w", sourcePath, err) + } + + tempFilePath := commons.MakeTargetLocalFilePath(encryptedFilename, put.encryptionFlagValues.TempPath) + + targetFilePath := commons.MakeTargetIRODSFilePath(put.filesystem, encryptedFilename, targetPath) + + return tempFilePath, targetFilePath, nil + } + + targetFilePath := commons.MakeTargetIRODSFilePath(put.filesystem, sourcePath, targetPath) + + return "", targetFilePath, nil +} + +func (put *PutCommand) encryptFile(sourcePath string, encryptedFilePath string, targetPath string, encryptionMode commons.EncryptionMode) (bool, error) { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "PutCommand", + "function": "encryptFile", + }) + + if encryptionMode != commons.EncryptionModeUnknown { + logger.Debugf("encrypt a file %q to %q", sourcePath, encryptedFilePath) + + encryptManager := put.getEncryptionManagerForEncryption(encryptionMode) + + err := encryptManager.EncryptFile(sourcePath, encryptedFilePath) + if err != nil { + return false, xerrors.Errorf("failed to encrypt %q to %q: %w", sourcePath, encryptedFilePath, err) } + + return true, nil } - return nil + return false, nil } diff --git a/cmd/subcmd/pwd.go b/cmd/subcmd/pwd.go index 91cd828..5b1482e 100644 --- a/cmd/subcmd/pwd.go +++ b/cmd/subcmd/pwd.go @@ -3,6 +3,8 @@ package subcmd import ( "fmt" + irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" "github.com/spf13/cobra" @@ -26,7 +28,31 @@ func AddPwdCommand(rootCmd *cobra.Command) { } func processPwdCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + pwd, err := NewPwdCommand(command, args) + if err != nil { + return err + } + + return pwd.Process() +} + +type PwdCommand struct { + command *cobra.Command + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem +} + +func NewPwdCommand(command *cobra.Command, args []string) (*PwdCommand, error) { + pwd := &PwdCommand{ + command: command, + } + + return pwd, nil +} + +func (pwd *PwdCommand) Process() error { + cont, err := flag.ProcessCommonFlags(pwd.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -41,11 +67,16 @@ func processPwdCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - printCurrentWorkingDir() + err = pwd.printCurrentWorkingDir() + if err != nil { + return xerrors.Errorf("failed to print current working directory: %w", err) + } return nil } -func printCurrentWorkingDir() { +func (pwd *PwdCommand) printCurrentWorkingDir() error { cwd := commons.GetCWD() fmt.Printf("%s\n", cwd) + + return nil } diff --git a/cmd/subcmd/rm.go b/cmd/subcmd/rm.go index 0bb190a..42df058 100644 --- a/cmd/subcmd/rm.go +++ b/cmd/subcmd/rm.go @@ -2,6 +2,7 @@ package subcmd import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -29,7 +30,42 @@ func AddRmCommand(rootCmd *cobra.Command) { } func processRmCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + rm, err := NewRmCommand(command, args) + if err != nil { + return err + } + + return rm.Process() +} + +type RmCommand struct { + command *cobra.Command + + recursiveFlagValues *flag.RecursiveFlagValues + forceFlagValues *flag.ForceFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + targetPaths []string +} + +func NewRmCommand(command *cobra.Command, args []string) (*RmCommand, error) { + rm := &RmCommand{ + command: command, + + recursiveFlagValues: flag.GetRecursiveFlagValues(), + forceFlagValues: flag.GetForceFlagValues(), + } + + // path + rm.targetPaths = args + + return rm, nil +} + +func (rm *RmCommand) Process() error { + cont, err := flag.ProcessCommonFlags(rm.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -44,30 +80,28 @@ func processRmCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - recursiveFlagValues := flag.GetRecursiveFlagValues() - forceFlagValues := flag.GetForceFlagValues() - // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + rm.account = commons.GetAccount() + rm.filesystem, err = commons.GetIRODSFSClient(rm.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer rm.filesystem.Release() - defer filesystem.Release() - - for _, sourcePath := range args { - err = removeOne(filesystem, sourcePath, forceFlagValues, recursiveFlagValues) + // remove + for _, targetPath := range rm.targetPaths { + err = rm.removeOne(targetPath) if err != nil { - return xerrors.Errorf("failed to perform rm %s: %w", sourcePath, err) + return xerrors.Errorf("failed to remove %q: %w", targetPath, err) } } return nil } -func removeOne(filesystem *irodsclient_fs.FileSystem, targetPath string, forceFlagValues *flag.ForceFlagValues, recursiveFlagValues *flag.RecursiveFlagValues) error { +func (rm *RmCommand) removeOne(targetPath string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "RmCommand", "function": "removeOne", }) @@ -76,35 +110,37 @@ func removeOne(filesystem *irodsclient_fs.FileSystem, targetPath string, forceFl zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - targetEntry, err := filesystem.Stat(targetPath) + targetEntry, err := rm.filesystem.Stat(targetPath) if err != nil { - //return xerrors.Errorf("failed to stat %s: %w", targetPath, err) - logger.Debugf("failed to find a data object %s, but trying to remove", targetPath) - err = filesystem.RemoveFile(targetPath, forceFlagValues.Force) + logger.Debugf("failed to find a data object %q, but trying to remove", targetPath) + err = rm.filesystem.RemoveFile(targetPath, rm.forceFlagValues.Force) if err != nil { - return xerrors.Errorf("failed to remove %s: %w", targetPath, err) + return xerrors.Errorf("failed to remove %q: %w", targetPath, err) } return nil } - if targetEntry.Type == irodsclient_fs.FileEntry { - // file - logger.Debugf("removing a data object %s", targetPath) - err = filesystem.RemoveFile(targetPath, forceFlagValues.Force) - if err != nil { - return xerrors.Errorf("failed to remove %s: %w", targetPath, err) - } - } else { + if targetEntry.IsDir() { // dir - if !recursiveFlagValues.Recursive { + if !rm.recursiveFlagValues.Recursive { return xerrors.Errorf("cannot remove a collection, recurse is not set") } - logger.Debugf("removing a collection %s", targetPath) - err = filesystem.RemoveDir(targetPath, recursiveFlagValues.Recursive, forceFlagValues.Force) + logger.Debugf("removing a collection %q", targetPath) + err = rm.filesystem.RemoveDir(targetPath, rm.recursiveFlagValues.Recursive, rm.forceFlagValues.Force) if err != nil { - return xerrors.Errorf("failed to remove dir %s: %w", targetPath, err) + return xerrors.Errorf("failed to remove a directory %q: %w", targetPath, err) } + + return nil + } + + // file + logger.Debugf("removing a data object %q", targetPath) + err = rm.filesystem.RemoveFile(targetPath, rm.forceFlagValues.Force) + if err != nil { + return xerrors.Errorf("failed to remove %q: %w", targetPath, err) } + return nil } diff --git a/cmd/subcmd/rmdir.go b/cmd/subcmd/rmdir.go index d14c371..359f2b8 100644 --- a/cmd/subcmd/rmdir.go +++ b/cmd/subcmd/rmdir.go @@ -2,6 +2,7 @@ package subcmd import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -22,11 +23,49 @@ func AddRmdirCommand(rootCmd *cobra.Command) { // attach common flags flag.SetCommonFlags(rmdirCmd, false) + flag.SetForceFlags(rmdirCmd, false) + flag.SetRecursiveFlags(rmdirCmd) + rootCmd.AddCommand(rmdirCmd) } func processRmdirCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + rm, err := NewRmDirCommand(command, args) + if err != nil { + return err + } + + return rm.Process() +} + +type RmDirCommand struct { + command *cobra.Command + + recursiveFlagValues *flag.RecursiveFlagValues + forceFlagValues *flag.ForceFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + targetPaths []string +} + +func NewRmDirCommand(command *cobra.Command, args []string) (*RmDirCommand, error) { + rmDir := &RmDirCommand{ + command: command, + + recursiveFlagValues: flag.GetRecursiveFlagValues(), + forceFlagValues: flag.GetForceFlagValues(), + } + + // path + rmDir.targetPaths = args + + return rmDir, nil +} + +func (rmDir *RmDirCommand) Process() error { + cont, err := flag.ProcessCommonFlags(rmDir.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -42,27 +81,29 @@ func processRmdirCommand(command *cobra.Command, args []string) error { } // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + rmDir.account = commons.GetAccount() + rmDir.filesystem, err = commons.GetIRODSFSClient(rmDir.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer rmDir.filesystem.Release() - defer filesystem.Release() - - for _, targetPath := range args { - err = removeDirOne(filesystem, targetPath) + // rmdir + for _, targetPath := range rmDir.targetPaths { + err = rmDir.removeOne(targetPath) if err != nil { - return xerrors.Errorf("failed to perform rmdir %s: %w", targetPath, err) + return xerrors.Errorf("failed to remove a directory %q: %w", targetPath, err) } } + return nil } -func removeDirOne(filesystem *irodsclient_fs.FileSystem, targetPath string) error { +func (rmDir *RmDirCommand) removeOne(targetPath string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "removeDirOne", + "struct": "RmDirCommand", + "function": "removeOne", }) cwd := commons.GetCWD() @@ -70,21 +111,22 @@ func removeDirOne(filesystem *irodsclient_fs.FileSystem, targetPath string) erro zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - targetEntry, err := filesystem.Stat(targetPath) + targetEntry, err := rmDir.filesystem.Stat(targetPath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", targetPath, err) + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } - if targetEntry.Type == irodsclient_fs.FileEntry { + if !targetEntry.IsDir() { // file - return xerrors.Errorf("%s is not a collection", targetPath) - } else { - // dir - logger.Debugf("removing a collection %s", targetPath) - err = filesystem.RemoveDir(targetPath, false, false) - if err != nil { - return xerrors.Errorf("failed to rmdir %s: %w", targetPath, err) - } + return commons.NewNotDirError(targetPath) + } + + // dir + logger.Debugf("removing a collection %q", targetPath) + err = rmDir.filesystem.RemoveDir(targetPath, rmDir.recursiveFlagValues.Recursive, rmDir.forceFlagValues.Force) + if err != nil { + return xerrors.Errorf("failed to remove a directory %q: %w", targetPath, err) } + return nil } diff --git a/cmd/subcmd/rmmeta.go b/cmd/subcmd/rmmeta.go index e272e5d..f5983db 100644 --- a/cmd/subcmd/rmmeta.go +++ b/cmd/subcmd/rmmeta.go @@ -4,6 +4,7 @@ import ( "strconv" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -30,12 +31,40 @@ func AddRmmetaCommand(rootCmd *cobra.Command) { } func processRmmetaCommand(command *cobra.Command, args []string) error { - logger := log.WithFields(log.Fields{ - "package": "subcmd", - "function": "processRmmetaCommand", - }) + rmMeta, err := NewRmMetaCommand(command, args) + if err != nil { + return err + } + + return rmMeta.Process() +} + +type RmMetaCommand struct { + command *cobra.Command + + targetObjectFlagValues *flag.TargetObjectFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + avuIDs []string +} + +func NewRmMetaCommand(command *cobra.Command, args []string) (*RmMetaCommand, error) { + rmMeta := &RmMetaCommand{ + command: command, + + targetObjectFlagValues: flag.GetTargetObjectFlagValues(command), + } - cont, err := flag.ProcessCommonFlags(command) + // path + rmMeta.avuIDs = args + + return rmMeta, nil +} + +func (rmMeta *RmMetaCommand) Process() error { + cont, err := flag.ProcessCommonFlags(rmMeta.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -50,175 +79,210 @@ func processRmmetaCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - targetObjectFlagValues := flag.GetTargetObjectFlagValues(command) - // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + rmMeta.account = commons.GetAccount() + rmMeta.filesystem, err = commons.GetIRODSFSClient(rmMeta.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer rmMeta.filesystem.Release() - defer filesystem.Release() - - for _, avuidString := range args { - if commons.IsDigitsOnly(avuidString) { - // avuid - avuid, err := strconv.ParseInt(avuidString, 10, 64) - if err != nil { - return xerrors.Errorf("failed to parse AVUID: %w", err) - } - - if targetObjectFlagValues.PathUpdated { - err = removeMetaFromPath(filesystem, targetObjectFlagValues.Path, avuid) - if err != nil { - return err - } - } else if targetObjectFlagValues.UserUpdated { - err = removeMetaFromUser(filesystem, targetObjectFlagValues.User, avuid) - if err != nil { - return err - } - } else if targetObjectFlagValues.ResourceUpdated { - err = removeMetaFromResource(filesystem, targetObjectFlagValues.Resource, avuid) - if err != nil { - return err - } - } else { - // nothing updated - return xerrors.Errorf("path, user, or resource must be given") - } - } else { - // possibly name - attr := avuidString - logger.Debugf("remove metadata with name %s", attr) - - if targetObjectFlagValues.PathUpdated { - err = removeMetaFromPathByName(filesystem, targetObjectFlagValues.Path, attr) - if err != nil { - return err - } - } else if targetObjectFlagValues.UserUpdated { - err = removeMetaFromUserByName(filesystem, targetObjectFlagValues.User, attr) - if err != nil { - return err - } - } else if targetObjectFlagValues.ResourceUpdated { - err = removeMetaFromResourceByName(filesystem, targetObjectFlagValues.Resource, attr) - if err != nil { - return err - } - } else { - // nothing updated - return xerrors.Errorf("path, user, or resource must be given") - } + // remove + for _, avuidString := range rmMeta.avuIDs { + err = rmMeta.removeOne(avuidString) + if err != nil { + return xerrors.Errorf("failed to remove meta for avuid (or name) %q: %w", avuidString, err) } - } + return nil } -func removeMetaFromPath(fs *irodsclient_fs.FileSystem, targetPath string, avuid int64) error { +func (rmMeta *RmMetaCommand) removeOne(avuidString string) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "RmMetaCommand", + "function": "removeOne", + }) + + if commons.IsDigitsOnly(avuidString) { + // avu ID + avuid, err := strconv.ParseInt(avuidString, 10, 64) + if err != nil { + return xerrors.Errorf("failed to parse AVUID: %w", err) + } + + return rmMeta.removeOneByID(avuid) + } + + // possibly name + logger.Debugf("remove metadata with name %q", avuidString) + return rmMeta.removeOneByName(avuidString) +} + +func (rmMeta *RmMetaCommand) removeOneByID(avuID int64) error { + if rmMeta.targetObjectFlagValues.PathUpdated { + err := rmMeta.removeMetaFromPath(rmMeta.targetObjectFlagValues.Path, avuID) + if err != nil { + return err + } + + return nil + } else if rmMeta.targetObjectFlagValues.UserUpdated { + err := rmMeta.removeMetaFromUser(rmMeta.targetObjectFlagValues.User, avuID) + if err != nil { + return err + } + + return nil + } else if rmMeta.targetObjectFlagValues.ResourceUpdated { + err := rmMeta.removeMetaFromResource(rmMeta.targetObjectFlagValues.Resource, avuID) + if err != nil { + return err + } + + return nil + } + + // nothing updated + return xerrors.Errorf("one of path, user, or resource must be selected") +} + +func (rmMeta *RmMetaCommand) removeOneByName(attrName string) error { + if rmMeta.targetObjectFlagValues.PathUpdated { + err := rmMeta.removeMetaFromPathByName(rmMeta.targetObjectFlagValues.Path, attrName) + if err != nil { + return err + } + + return nil + } else if rmMeta.targetObjectFlagValues.UserUpdated { + err := rmMeta.removeMetaFromUserByName(rmMeta.targetObjectFlagValues.User, attrName) + if err != nil { + return err + } + + return nil + } else if rmMeta.targetObjectFlagValues.ResourceUpdated { + err := rmMeta.removeMetaFromResourceByName(rmMeta.targetObjectFlagValues.Resource, attrName) + if err != nil { + return err + } + + return nil + } + + // nothing updated + return xerrors.Errorf("one of path, user, or resource must be selected") +} + +func (rmMeta *RmMetaCommand) removeMetaFromPath(targetPath string, avuid int64) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "RmMetaCommand", "function": "removeMetaFromPath", }) - logger.Debugf("remove metadata %d from path %s", avuid, targetPath) + logger.Debugf("remove metadata %d from path %q", avuid, targetPath) cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - err := fs.DeleteMetadata(targetPath, avuid) + err := rmMeta.filesystem.DeleteMetadata(targetPath, avuid) if err != nil { - return xerrors.Errorf("failed to delete metadata %d from path %s: %w", avuid, targetPath, err) + return xerrors.Errorf("failed to delete metadata %d from path %q: %w", avuid, targetPath, err) } return nil } -func removeMetaFromUser(fs *irodsclient_fs.FileSystem, username string, avuid int64) error { +func (rmMeta *RmMetaCommand) removeMetaFromUser(username string, avuid int64) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "RmMetaCommand", "function": "removeMetaFromUser", }) - logger.Debugf("remove metadata %d from user %s", avuid, username) + logger.Debugf("remove metadata %d from user %q", avuid, username) - err := fs.DeleteUserMetadata(username, avuid) + err := rmMeta.filesystem.DeleteUserMetadata(username, avuid) if err != nil { - return xerrors.Errorf("failed to delete metadata %d from user %s: %w", avuid, username, err) + return xerrors.Errorf("failed to delete metadata %d from user %q: %w", avuid, username, err) } return nil } -func removeMetaFromResource(fs *irodsclient_fs.FileSystem, resource string, avuid int64) error { +func (rmMeta *RmMetaCommand) removeMetaFromResource(resource string, avuid int64) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "RmMetaCommand", "function": "removeMetaFromResource", }) - logger.Debugf("remove metadata %d from resource %s", avuid, resource) + logger.Debugf("remove metadata %d from resource %q", avuid, resource) - err := fs.DeleteResourceMetadata(resource, avuid) + err := rmMeta.filesystem.DeleteResourceMetadata(resource, avuid) if err != nil { - return xerrors.Errorf("failed to delete metadata %d from resource %s: %w", avuid, resource, err) + return xerrors.Errorf("failed to delete metadata %d from resource %q: %w", avuid, resource, err) } return nil } -func removeMetaFromPathByName(fs *irodsclient_fs.FileSystem, targetPath string, attr string) error { +func (rmMeta *RmMetaCommand) removeMetaFromPathByName(targetPath string, attr string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "RmMetaCommand", "function": "removeMetaFromPathByName", }) - logger.Debugf("remove metadata %s from path %s by name", attr, targetPath) + logger.Debugf("remove metadata %q from path %q by name", attr, targetPath) cwd := commons.GetCWD() home := commons.GetHomeDir() zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - err := fs.DeleteMetadataByName(targetPath, attr) + err := rmMeta.filesystem.DeleteMetadataByName(targetPath, attr) if err != nil { - return xerrors.Errorf("failed to delete metadata %s from path %s by name: %w", attr, targetPath, err) + return xerrors.Errorf("failed to delete metadata %q from path %q by name: %w", attr, targetPath, err) } return nil } -func removeMetaFromUserByName(fs *irodsclient_fs.FileSystem, username string, attr string) error { +func (rmMeta *RmMetaCommand) removeMetaFromUserByName(username string, attr string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "RmMetaCommand", "function": "removeMetaFromUserByName", }) - logger.Debugf("remove metadata %s from user %s by name", attr, username) + logger.Debugf("remove metadata %q from user %q by name", attr, username) - err := fs.DeleteUserMetadataByName(username, attr) + err := rmMeta.filesystem.DeleteUserMetadataByName(username, attr) if err != nil { - return xerrors.Errorf("failed to delete metadata %s from user %s by name: %w", attr, username, err) + return xerrors.Errorf("failed to delete metadata %q from user %q by name: %w", attr, username, err) } return nil } -func removeMetaFromResourceByName(fs *irodsclient_fs.FileSystem, resource string, attr string) error { +func (rmMeta *RmMetaCommand) removeMetaFromResourceByName(resource string, attr string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "RmMetaCommand", "function": "removeMetaFromResourceByName", }) - logger.Debugf("remove metadata %s from resource %s by name", attr, resource) + logger.Debugf("remove metadata %q from resource %q by name", attr, resource) - err := fs.DeleteResourceMetadataByName(resource, attr) + err := rmMeta.filesystem.DeleteResourceMetadataByName(resource, attr) if err != nil { - return xerrors.Errorf("failed to delete metadata %s from resource %s by name: %w", attr, resource, err) + return xerrors.Errorf("failed to delete metadata %q from resource %q by name: %w", attr, resource, err) } return nil diff --git a/cmd/subcmd/rmticket.go b/cmd/subcmd/rmticket.go index 0845a3c..eb5f3d2 100644 --- a/cmd/subcmd/rmticket.go +++ b/cmd/subcmd/rmticket.go @@ -2,6 +2,7 @@ package subcmd import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -26,7 +27,36 @@ func AddRmticketCommand(rootCmd *cobra.Command) { } func processRmticketCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + rmTicket, err := NewRmTicketCommand(command, args) + if err != nil { + return err + } + + return rmTicket.Process() +} + +type RmTicketCommand struct { + command *cobra.Command + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + tickets []string +} + +func NewRmTicketCommand(command *cobra.Command, args []string) (*RmTicketCommand, error) { + rmTicket := &RmTicketCommand{ + command: command, + } + + // tickets + rmTicket.tickets = args + + return rmTicket, nil +} + +func (rmTicket *RmTicketCommand) Process() error { + cont, err := flag.ProcessCommonFlags(rmTicket.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -42,16 +72,15 @@ func processRmticketCommand(command *cobra.Command, args []string) error { } // Create a file system - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + rmTicket.account = commons.GetAccount() + rmTicket.filesystem, err = commons.GetIRODSFSClient(rmTicket.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer rmTicket.filesystem.Release() - defer filesystem.Release() - - for _, ticketName := range args { - err = removeTicket(filesystem, ticketName) + for _, ticketName := range rmTicket.tickets { + err = rmTicket.removeTicket(ticketName) if err != nil { return err } @@ -59,17 +88,18 @@ func processRmticketCommand(command *cobra.Command, args []string) error { return nil } -func removeTicket(fs *irodsclient_fs.FileSystem, ticketName string) error { +func (rmTicket *RmTicketCommand) removeTicket(ticketName string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", + "struct": "RmTicketCommand", "function": "removeTicket", }) - logger.Debugf("remove ticket: %s", ticketName) + logger.Debugf("remove ticket %q", ticketName) - err := fs.DeleteTicket(ticketName) + err := rmTicket.filesystem.DeleteTicket(ticketName) if err != nil { - return xerrors.Errorf("failed to delete ticket %s: %w", ticketName, err) + return xerrors.Errorf("failed to delete ticket %q: %w", ticketName, err) } return nil diff --git a/cmd/subcmd/svrinfo.go b/cmd/subcmd/svrinfo.go index 6fa88da..af207a4 100644 --- a/cmd/subcmd/svrinfo.go +++ b/cmd/subcmd/svrinfo.go @@ -4,7 +4,7 @@ import ( "os" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" - "github.com/cyverse/go-irodsclient/irods/types" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" "github.com/jedib0t/go-pretty/v6/table" @@ -30,7 +30,31 @@ func AddSvrinfoCommand(rootCmd *cobra.Command) { } func processSvrinfoCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + svrInfo, err := NewSvrInfoCommand(command, args) + if err != nil { + return err + } + + return svrInfo.Process() +} + +type SvrInfoCommand struct { + command *cobra.Command + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem +} + +func NewSvrInfoCommand(command *cobra.Command, args []string) (*SvrInfoCommand, error) { + svrInfo := &SvrInfoCommand{ + command: command, + } + + return svrInfo, nil +} + +func (svrInfo *SvrInfoCommand) Process() error { + cont, err := flag.ProcessCommonFlags(svrInfo.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -45,32 +69,33 @@ func processSvrinfoCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - // Create a connection - account := commons.GetAccount() - filesystem, err := commons.GetIRODSFSClient(account) + // Create a file system + svrInfo.account = commons.GetAccount() + svrInfo.filesystem, err = commons.GetIRODSFSClient(svrInfo.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) } + defer svrInfo.filesystem.Release() - defer filesystem.Release() - - err = displayVersion(account, filesystem) + // run + err = svrInfo.displayInfo() if err != nil { - return xerrors.Errorf("failed to perform svrinfo: %w", err) + return xerrors.Errorf("failed to display server info: %w", err) } return nil } -func displayVersion(account *types.IRODSAccount, fs *irodsclient_fs.FileSystem) error { +func (svrInfo *SvrInfoCommand) displayInfo() error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "displayVersion", + "struct": "SvrInfoCommand", + "function": "displayInfo", }) logger.Debug("displaying version") - ver, err := fs.GetServerVersion() + ver, err := svrInfo.filesystem.GetServerVersion() if err != nil { return xerrors.Errorf("failed to get server version: %w", err) } @@ -89,7 +114,7 @@ func displayVersion(account *types.IRODSAccount, fs *irodsclient_fs.FileSystem) }, { "iRODS Zone", - account.ClientZone, + svrInfo.account.ClientZone, }, }, table.RowConfig{}) t.Render() diff --git a/cmd/subcmd/sync.go b/cmd/subcmd/sync.go index 53c82ed..b90577e 100644 --- a/cmd/subcmd/sync.go +++ b/cmd/subcmd/sync.go @@ -4,6 +4,8 @@ import ( "os" "strings" + irodsclient_fs "github.com/cyverse/go-irodsclient/fs" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -39,7 +41,42 @@ func AddSyncCommand(rootCmd *cobra.Command) { } func processSyncCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + sync, err := NewSyncCommand(command, args) + if err != nil { + return err + } + + return sync.Process() +} + +type SyncCommand struct { + command *cobra.Command + + retryFlagValues *flag.RetryFlagValues + + account *irodsclient_types.IRODSAccount + filesystem *irodsclient_fs.FileSystem + + sourcePaths []string + targetPath string +} + +func NewSyncCommand(command *cobra.Command, args []string) (*SyncCommand, error) { + sync := &SyncCommand{ + command: command, + + retryFlagValues: flag.GetRetryFlagValues(), + } + + // path + sync.sourcePaths = args[:len(args)-1] + sync.targetPath = args[len(args)-1] + + return sync, nil +} + +func (sync *SyncCommand) Process() error { + cont, err := flag.ProcessCommonFlags(sync.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -54,116 +91,85 @@ func processSyncCommand(command *cobra.Command, args []string) error { return xerrors.Errorf("failed to input missing fields: %w", err) } - retryFlagValues := flag.GetRetryFlagValues() - - if retryFlagValues.RetryNumber > 0 && !retryFlagValues.RetryChild { - err = commons.RunWithRetry(retryFlagValues.RetryNumber, retryFlagValues.RetryIntervalSeconds) + // handle retry + if sync.retryFlagValues.RetryNumber > 0 && !sync.retryFlagValues.RetryChild { + err = commons.RunWithRetry(sync.retryFlagValues.RetryNumber, sync.retryFlagValues.RetryIntervalSeconds) if err != nil { - return xerrors.Errorf("failed to run with retry %d: %w", retryFlagValues.RetryNumber, err) + return xerrors.Errorf("failed to run with retry %d: %w", sync.retryFlagValues.RetryNumber, err) } return nil } - targetPath := args[len(args)-1] - sourcePaths := args[:len(args)-1] - - localSources := []string{} - irodsSources := []string{} + localSourcePaths := []string{} + irodsSourcePaths := []string{} - for _, sourcePath := range sourcePaths { + for _, sourcePath := range sync.sourcePaths { if strings.HasPrefix(sourcePath, "i:") { - irodsSources = append(irodsSources, sourcePath[2:]) + // irods + irodsSourcePaths = append(irodsSourcePaths, sourcePath[2:]) } else { - localSources = append(localSources, sourcePath) + // local + localSourcePaths = append(localSourcePaths, sourcePath) } } - if len(localSources) > 0 { - // source is local - if !strings.HasPrefix(targetPath, "i:") { - // local to local - return xerrors.Errorf("syncing local to local is not supported") - } - - // local to iRODS - // target must starts with "i:" - err := syncFromLocalToIRODS(command) + if len(localSourcePaths) > 0 { + err := sync.syncLocal(sync.targetPath) if err != nil { - return xerrors.Errorf("failed to perform sync (from local to iRODS): %w", err) + return xerrors.Errorf("failed to sync (from local): %w", err) } } - if len(irodsSources) > 0 { - // source is iRODS - if strings.HasPrefix(targetPath, "i:") { - // iRODS to iRODS - err := syncFromIRODSToIRODS(command, irodsSources, targetPath[2:]) - if err != nil { - return xerrors.Errorf("failed to perform sync (from iRODS to iRODS): %w", err) - } - } else { - // iRODS to local - err := syncFromIRODSToLocal(command, irodsSources, targetPath) - if err != nil { - return xerrors.Errorf("failed to perform sync (from iRODS to local): %w", err) - } + if len(irodsSourcePaths) > 0 { + err := sync.syncIRODS(sync.targetPath) + if err != nil { + return xerrors.Errorf("failed to sync (from iRODS): %w", err) } } return nil } -func syncFromLocalToIRODS(command *cobra.Command) error { - logger := log.WithFields(log.Fields{ - "package": "subcmd", - "function": "syncFromLocalToIRODS", - }) +func (sync *SyncCommand) syncLocal(targetPath string) error { + if !strings.HasPrefix(targetPath, "i:") { + // local to local + return xerrors.Errorf("syncing local to local is not supported") + } - newArgs := []string{} + // local to iRODS + // target must starts with "i:" + err := sync.syncLocalToIRODS() + if err != nil { + return xerrors.Errorf("failed to sync (from local to iRODS): %w", err) + } - commandName := command.CalledAs() - commandIdx := -1 + return nil +} - osArgs := os.Args[1:] - for argIdx, arg := range osArgs { - if arg == commandName { - commandIdx = argIdx - break +func (sync *SyncCommand) syncIRODS(targetPath string) error { + if strings.HasPrefix(targetPath, "i:") { + // iRODS to iRODS + err := sync.syncIRODSToIRODS() + if err != nil { + return xerrors.Errorf("failed to sync (from iRODS to iRODS): %w", err) } - } - if commandIdx < 0 { - return xerrors.Errorf("failed to find command location") + return nil } - newArgs = append(newArgs, osArgs[:commandIdx]...) - newArgs = append(newArgs, "--diff") - newArgs = append(newArgs, osArgs[commandIdx+1:]...) - - // filter out retry flag - newArgs2 := []string{} - for _, arg := range newArgs { - if arg != "--retry_child" { - newArgs2 = append(newArgs2, arg) - } + // iRODS to local + err := sync.syncIRODSToLocal() + if err != nil { + return xerrors.Errorf("failed to sync (from iRODS to local): %w", err) } - // run bput - logger.Debugf("run bput with args: %v", newArgs2) - bputCmd.ParseFlags(newArgs2) - argWoFlags := bputCmd.Flags().Args() - return bputCmd.RunE(bputCmd, argWoFlags) + return nil } -func syncFromIRODSToIRODS(command *cobra.Command, sourcePaths []string, targetPath string) error { - logger := log.WithFields(log.Fields{ - "package": "subcmd", - "function": "syncFromIRODSToIRODS", - }) - +func (sync *SyncCommand) getNewCommandArgsForRetry() ([]string, error) { newArgs := []string{} - commandName := command.CalledAs() + commandName := sync.command.CalledAs() commandIdx := -1 osArgs := os.Args[1:] @@ -175,7 +181,7 @@ func syncFromIRODSToIRODS(command *cobra.Command, sourcePaths []string, targetPa } if commandIdx < 0 { - return xerrors.Errorf("failed to find command location") + return nil, xerrors.Errorf("failed to find command location") } newArgs = append(newArgs, osArgs[:commandIdx]...) @@ -183,58 +189,69 @@ func syncFromIRODSToIRODS(command *cobra.Command, sourcePaths []string, targetPa newArgs = append(newArgs, osArgs[commandIdx+1:]...) // filter out retry flag - newArgs2 := []string{} + newArgsWoRetryFlag := []string{} for _, arg := range newArgs { if arg != "--retry_child" { - newArgs2 = append(newArgs2, arg) + newArgsWoRetryFlag = append(newArgsWoRetryFlag, arg) } } - // run bput - logger.Debugf("run cp with args: %v", newArgs2) - cpCmd.ParseFlags(newArgs2) - argWoFlags := cpCmd.Flags().Args() - return cpCmd.RunE(cpCmd, argWoFlags) + return newArgsWoRetryFlag, nil } -func syncFromIRODSToLocal(command *cobra.Command, sourcePaths []string, targetPath string) error { +func (sync *SyncCommand) syncLocalToIRODS() error { logger := log.WithFields(log.Fields{ "package": "subcmd", - "function": "syncFromIRODSToLocal", + "struct": "SyncCommand", + "function": "syncLocalToIRODS", }) - newArgs := []string{} + newArgs, err := sync.getNewCommandArgsForRetry() + if err != nil { + return xerrors.Errorf("failed to get new command args for retry: %w", err) + } - commandName := command.CalledAs() - commandIdx := -1 + // run bput + logger.Debugf("run bput with args: %v", newArgs) + bputCmd.ParseFlags(newArgs) + argWoFlags := bputCmd.Flags().Args() + return bputCmd.RunE(bputCmd, argWoFlags) +} - osArgs := os.Args[1:] - for argIdx, arg := range osArgs { - if arg == commandName { - commandIdx = argIdx - break - } - } +func (sync *SyncCommand) syncIRODSToIRODS() error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "SyncCommand", + "function": "syncIRODSToIRODS", + }) - if commandIdx < 0 { - return xerrors.Errorf("failed to find command location") + newArgs, err := sync.getNewCommandArgsForRetry() + if err != nil { + return xerrors.Errorf("failed to get new command args for retry: %w", err) } - newArgs = append(newArgs, osArgs[:commandIdx]...) - newArgs = append(newArgs, "--diff") - newArgs = append(newArgs, osArgs[commandIdx+1:]...) + // run cp + logger.Debugf("run cp with args: %v", newArgs) + cpCmd.ParseFlags(newArgs) + argWoFlags := cpCmd.Flags().Args() + return cpCmd.RunE(cpCmd, argWoFlags) +} - // filter out retry flag - newArgs2 := []string{} - for _, arg := range newArgs { - if arg != "--retry_child" { - newArgs2 = append(newArgs2, arg) - } +func (sync *SyncCommand) syncIRODSToLocal() error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "SyncCommand", + "function": "syncIRODSToLocal", + }) + + newArgs, err := sync.getNewCommandArgsForRetry() + if err != nil { + return xerrors.Errorf("failed to get new command args for retry: %w", err) } - // run bput - logger.Debugf("run get with args: %v", newArgs2) - getCmd.ParseFlags(newArgs2) + // run get + logger.Debugf("run get with args: %v", newArgs) + getCmd.ParseFlags(newArgs) argWoFlags := getCmd.Flags().Args() return getCmd.RunE(getCmd, argWoFlags) } diff --git a/cmd/subcmd/upgrade.go b/cmd/subcmd/upgrade.go index 16ffc24..9534386 100644 --- a/cmd/subcmd/upgrade.go +++ b/cmd/subcmd/upgrade.go @@ -28,7 +28,32 @@ func AddUpgradeCommand(rootCmd *cobra.Command) { } func processUpgradeCommand(command *cobra.Command, args []string) error { - cont, err := flag.ProcessCommonFlags(command) + upgrade, err := NewUpgradeCommand(command, args) + if err != nil { + return err + } + + return upgrade.Process() +} + +type UpgradeCommand struct { + command *cobra.Command + + checkVersionFlagValues *flag.CheckVersionFlagValues +} + +func NewUpgradeCommand(command *cobra.Command, args []string) (*UpgradeCommand, error) { + upgrade := &UpgradeCommand{ + command: command, + + checkVersionFlagValues: flag.GetCheckVersionFlagValues(), + } + + return upgrade, nil +} + +func (upgrade *UpgradeCommand) Process() error { + cont, err := flag.ProcessCommonFlags(upgrade.command) if err != nil { return xerrors.Errorf("failed to process common flags: %w", err) } @@ -37,24 +62,24 @@ func processUpgradeCommand(command *cobra.Command, args []string) error { return nil } - checkVersionFlagValues := flag.GetCheckVersionFlagValues() - - if checkVersionFlagValues.Check { - err = checkNewVersion() + if upgrade.checkVersionFlagValues.Check { + err = upgrade.checkNewVersion() if err != nil { return xerrors.Errorf("failed to check new release: %w", err) } - } else { - err = upgradeToNewVersion() - if err != nil { - return xerrors.Errorf("failed to upgrade to new release: %w", err) - } + + return nil + } + + err = upgrade.upgrade() + if err != nil { + return xerrors.Errorf("failed to upgrade to new release: %w", err) } return nil } -func checkNewVersion() error { +func (upgrade *UpgradeCommand) checkNewVersion() error { newRelease, err := commons.CheckNewRelease() if err != nil { return err @@ -68,6 +93,6 @@ func checkNewVersion() error { return nil } -func upgradeToNewVersion() error { +func (upgrade *UpgradeCommand) upgrade() error { return commons.SelfUpgrade() } diff --git a/commons/bundle_transfer.go b/commons/bundle_transfer.go index 2a0c989..d72d8ff 100644 --- a/commons/bundle_transfer.go +++ b/commons/bundle_transfer.go @@ -102,7 +102,7 @@ func (bundle *Bundle) GetBundleFilename() (string, error) { func (bundle *Bundle) AddFile(localPath string, size int64) error { irodsPath, err := bundle.manager.getTargetPath(localPath) if err != nil { - return xerrors.Errorf("failed to get target path for %s: %w", localPath, err) + return xerrors.Errorf("failed to get target path for %q: %w", localPath, err) } e := &BundleEntry{ @@ -126,7 +126,7 @@ func (bundle *Bundle) AddFile(localPath string, size int64) error { func (bundle *Bundle) AddDir(localPath string) error { irodsPath, err := bundle.manager.getTargetPath(localPath) if err != nil { - return xerrors.Errorf("failed to get target path for %s: %w", localPath, err) + return xerrors.Errorf("failed to get target path for %q: %w", localPath, err) } e := &BundleEntry{ @@ -205,7 +205,12 @@ type BundleTransferManager struct { } // NewBundleTransferManager creates a new BundleTransferManager -func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, irodsDestPath string, maxBundleFileNum int, maxBundleFileSize int64, singleThreaded bool, uploadThreadNum int, redirectToResource bool, useIcat bool, localTempDirPath string, irodsTempDirPath string, diff bool, noHash bool, noBulkReg bool, showProgress bool, showFullPath bool) *BundleTransferManager { +func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, irodsDestPath string, bundleRootPath string, maxBundleFileNum int, maxBundleFileSize int64, singleThreaded bool, uploadThreadNum int, redirectToResource bool, useIcat bool, localTempDirPath string, irodsTempDirPath string, diff bool, noHash bool, noBulkReg bool, showProgress bool, showFullPath bool) *BundleTransferManager { + cwd := GetCWD() + home := GetHomeDir() + zone := GetZone() + irodsDestPath = MakeIRODSPath(cwd, home, zone, irodsDestPath) + manager := &BundleTransferManager{ filesystem: fs, irodsDestPath: irodsDestPath, @@ -268,7 +273,7 @@ func (manager *BundleTransferManager) progress(name string, processed int64, tot func (manager *BundleTransferManager) getTargetPath(localPath string) (string, error) { relPath, err := filepath.Rel(manager.bundleRootPath, localPath) if err != nil { - return "", xerrors.Errorf("failed to compute relative path %s to %s: %w", localPath, manager.bundleRootPath, err) + return "", xerrors.Errorf("failed to compute relative path %q to %q: %w", localPath, manager.bundleRootPath, err) } return path.Join(manager.irodsDestPath, filepath.ToSlash(relPath)), nil @@ -309,7 +314,7 @@ func (manager *BundleTransferManager) Schedule(source string, dir bool, size int // add new bundle, err := newBundle(manager) if err != nil { - return xerrors.Errorf("failed to create a new bundle for %s: %w", source, err) + return xerrors.Errorf("failed to create a new bundle for %q: %w", source, err) } manager.currentBundle = bundle @@ -319,77 +324,77 @@ func (manager *BundleTransferManager) Schedule(source string, dir bool, size int defer manager.mutex.Unlock() targePath, err := manager.getTargetPath(source) if err != nil { - return xerrors.Errorf("failed to get target path for %s: %w", source, err) + return xerrors.Errorf("failed to get target path for %q: %w", source, err) } MarkPathMap(manager.inputPathMap, targePath) if manager.differentFilesOnly { - logger.Debugf("checking if target %s for source %s exists", targePath, source) + logger.Debugf("checking if target %q for source %q exists", targePath, source) if dir { // handle dir exist := manager.filesystem.ExistsDir(targePath) if exist { - Printf("skip adding a dir %s to the bundle. The dir already exists!\n", source) - logger.Debugf("skip adding a dir %s to the bundle. The dir already exists!", source) + Printf("skip adding a directory %q to the bundle. The dir already exists!\n", source) + logger.Debugf("skip adding a directory %q to the bundle. The dir already exists!", source) return nil } - logger.Debugf("adding a dir %s to the bundle as it doesn't exist", source) + logger.Debugf("adding a directory %q to the bundle as it doesn't exist", source) } else { exist := manager.filesystem.ExistsFile(targePath) if exist { targetEntry, err := manager.filesystem.Stat(targePath) if err != nil { - return xerrors.Errorf("failed to stat %s: %w", targePath, err) + return xerrors.Errorf("failed to stat %q: %w", targePath, err) } if manager.noHashForComparison { if targetEntry.Size == size { - Printf("skip adding a file %s to the bundle. The file already exists!\n", source) - logger.Debugf("skip adding a file %s to the bundle. The file already exists!", source) + Printf("skip adding a file %q to the bundle. The file already exists!\n", source) + logger.Debugf("skip adding a file %q to the bundle. The file already exists!", source) return nil } - logger.Debugf("adding a file %s to the bundle as it has different size %d != %d", source, targetEntry.Size, size) + logger.Debugf("adding a file %q to the bundle as it has different size %d != %d", source, targetEntry.Size, size) } else { if targetEntry.Size == size { if len(targetEntry.CheckSum) > 0 { // compare hash hash, err := irodsclient_util.HashLocalFile(source, string(targetEntry.CheckSumAlgorithm)) if err != nil { - return xerrors.Errorf("failed to get hash %s: %w", source, err) + return xerrors.Errorf("failed to get hash %q: %w", source, err) } if bytes.Equal(hash, targetEntry.CheckSum) { - Printf("skip adding a file %s to the bundle. The file with the same hash already exists!\n", source) - logger.Debugf("skip adding a file %s to the bundle. The file with the same hash already exists!", source) + Printf("skip adding a file %q to the bundle. The file with the same hash already exists!\n", source) + logger.Debugf("skip adding a file %q to the bundle. The file with the same hash already exists!", source) return nil } - logger.Debugf("adding a file %s to the bundle as it has different hash, %s vs %s (alg %s)", source, hash, targetEntry.CheckSum, targetEntry.CheckSumAlgorithm) + logger.Debugf("adding a file %q to the bundle as it has different hash, %q vs %q (alg %q)", source, hash, targetEntry.CheckSum, targetEntry.CheckSumAlgorithm) } else { - logger.Debugf("adding a file %s to the bundle as the file in iRODS doesn't have hash yet", source) + logger.Debugf("adding a file %q to the bundle as the file in iRODS doesn't have hash yet", source) } } else { - logger.Debugf("adding a file %s to the bundle as it has different size %d != %d", source, targetEntry.Size, size) + logger.Debugf("adding a file %q to the bundle as it has different size %d != %d", source, targetEntry.Size, size) } } } else { - logger.Debugf("adding a file %s to the bundle as it doesn't exist", source) + logger.Debugf("adding a file %q to the bundle as it doesn't exist", source) } } } if dir { manager.currentBundle.AddDir(source) - logger.Debugf("> scheduled a local file bundle-upload %s", source) + logger.Debugf("> scheduled a local file bundle-upload %q", source) return nil } manager.currentBundle.AddFile(source, size) - logger.Debugf("> scheduled a local file bundle-upload %s", source) + logger.Debugf("> scheduled a local file bundle-upload %q", source) return nil } @@ -444,10 +449,6 @@ func (manager *BundleTransferManager) Wait() error { return nil } -func (manager *BundleTransferManager) SetBundleRootPath(bundleRootPath string) { - manager.bundleRootPath = bundleRootPath -} - func (manager *BundleTransferManager) CleanUpBundles() { logger := log.WithFields(log.Fields{ "package": "commons", @@ -455,12 +456,12 @@ func (manager *BundleTransferManager) CleanUpBundles() { "function": "CleanUpBundles", }) - logger.Debugf("clearing bundle files in %s", manager.irodsTempDirPath) + logger.Debugf("clearing bundle files in %q", manager.irodsTempDirPath) // if the staging dir is not in target path entries, err := manager.filesystem.List(manager.irodsTempDirPath) if err != nil { - logger.WithError(err).Warnf("failed to listing staging dir %s", manager.irodsTempDirPath) + logger.WithError(err).Warnf("failed to listing a staging directory %q", manager.irodsTempDirPath) return } @@ -469,7 +470,7 @@ func (manager *BundleTransferManager) CleanUpBundles() { // remove the dir err := manager.filesystem.RemoveDir(manager.irodsTempDirPath, true, true) if err != nil { - logger.WithError(err).Warnf("failed to remove staging dir %s", manager.irodsTempDirPath) + logger.WithError(err).Warnf("failed to remove staging directory %q", manager.irodsTempDirPath) return } return @@ -479,13 +480,13 @@ func (manager *BundleTransferManager) CleanUpBundles() { // ask deleted := 0 for _, entry := range entries { - del := InputYN(fmt.Sprintf("removing old bundle file %s found. Delete?", entry.Path)) + del := InputYN(fmt.Sprintf("removing old bundle file %q found. Delete?", entry.Path)) if del { - logger.Debugf("deleting old bundle file %s", entry.Path) + logger.Debugf("deleting old bundle file %q", entry.Path) removeErr := os.Remove(entry.Path) if removeErr != nil { - logger.WithError(removeErr).Warnf("failed to remove old bundle file %s", entry.Path) + logger.WithError(removeErr).Warnf("failed to remove old bundle file %q", entry.Path) } deleted++ @@ -496,7 +497,7 @@ func (manager *BundleTransferManager) CleanUpBundles() { // if the staging dir is not in target path entries, err = manager.filesystem.List(manager.irodsTempDirPath) if err != nil { - logger.WithError(err).Warnf("failed to listing staging dir %s", manager.irodsTempDirPath) + logger.WithError(err).Warnf("failed to listing a staging directory %q", manager.irodsTempDirPath) return } @@ -505,7 +506,7 @@ func (manager *BundleTransferManager) CleanUpBundles() { // remove the dir err := manager.filesystem.RemoveDir(manager.irodsTempDirPath, true, true) if err != nil { - logger.WithError(err).Warnf("failed to remove staging dir %s", manager.irodsTempDirPath) + logger.WithError(err).Warnf("failed to remove a staging directory %q", manager.irodsTempDirPath) return } return @@ -728,8 +729,8 @@ func (manager *BundleTransferManager) Start() { // process bundle - remove stale files and create new dirs go func() { - logger.Debug("start stale file remove and dir create thread") - defer logger.Debug("exit stale file remove and dir create thread") + logger.Debug("start stale file remove and directory create thread") + defer logger.Debug("exit stale file remove and directory create thread") defer close(processBundleExtractChan2) @@ -920,14 +921,14 @@ func (manager *BundleTransferManager) processBundleRemoveFilesAndMakeDirs(bundle } logger.Error(err) - return xerrors.Errorf("failed to stat data object or collection %s", bundleEntry.IRODSPath) + return xerrors.Errorf("failed to stat data object or collection %q", bundleEntry.IRODSPath) } } if entry != nil { if entry.IsDir() { if !bundleEntry.Dir { - logger.Debugf("deleting exising collection %s", bundleEntry.IRODSPath) + logger.Debugf("deleting exising collection %q", bundleEntry.IRODSPath) err := manager.filesystem.RemoveDir(bundleEntry.IRODSPath, true, true) if err != nil { if manager.showProgress { @@ -935,12 +936,12 @@ func (manager *BundleTransferManager) processBundleRemoveFilesAndMakeDirs(bundle } logger.Error(err) - return xerrors.Errorf("failed to delete existing collection %s", bundleEntry.IRODSPath) + return xerrors.Errorf("failed to delete existing collection %q", bundleEntry.IRODSPath) } } } else { // file - logger.Debugf("deleting exising data object %s", bundleEntry.IRODSPath) + logger.Debugf("deleting exising data object %q", bundleEntry.IRODSPath) err := manager.filesystem.RemoveFile(bundleEntry.IRODSPath, true) if err != nil { @@ -949,7 +950,7 @@ func (manager *BundleTransferManager) processBundleRemoveFilesAndMakeDirs(bundle } logger.Error(err) - return xerrors.Errorf("failed to delete existing data object %s", bundleEntry.IRODSPath) + return xerrors.Errorf("failed to delete existing data object %q", bundleEntry.IRODSPath) } } } @@ -971,7 +972,7 @@ func (manager *BundleTransferManager) processBundleTar(bundle *Bundle) error { "function": "processBundleTar", }) - logger.Debugf("creating a tarball for bundle %d to %s", bundle.index, bundle.localBundlePath) + logger.Debugf("creating a tarball for bundle %d to %q", bundle.index, bundle.localBundlePath) progressName := manager.getProgressName(bundle, BundleTaskNameTar) @@ -994,7 +995,7 @@ func (manager *BundleTransferManager) processBundleTar(bundle *Bundle) error { manager.progress(progressName, totalFileNum, totalFileNum, progress.UnitsDefault, false) } - logger.Debugf("skip - creating a tarball for bundle %d to %s", bundle.index, bundle.localBundlePath) + logger.Debugf("skip - creating a tarball for bundle %d to %q", bundle.index, bundle.localBundlePath) return nil } @@ -1009,10 +1010,10 @@ func (manager *BundleTransferManager) processBundleTar(bundle *Bundle) error { manager.progress(progressName, 0, totalFileNum, progress.UnitsDefault, true) } - return xerrors.Errorf("failed to create a tarball for bundle %d to %s: %w", bundle.index, bundle.localBundlePath, err) + return xerrors.Errorf("failed to create a tarball for bundle %d to %q: %w", bundle.index, bundle.localBundlePath, err) } - logger.Debugf("created a tarball for bundle %d to %s", bundle.index, bundle.localBundlePath) + logger.Debugf("created a tarball for bundle %d to %q", bundle.index, bundle.localBundlePath) return nil } @@ -1023,7 +1024,7 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error "function": "processBundleUpload", }) - logger.Debugf("uploading bundle %d to %s", bundle.index, bundle.irodsBundlePath) + logger.Debugf("uploading bundle %d to %q", bundle.index, bundle.irodsBundlePath) progressName := manager.getProgressName(bundle, BundleTaskNameUpload) @@ -1046,7 +1047,7 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error manager.progress(progressName, -1, totalFileSize, progress.UnitsBytes, true) } - return xerrors.Errorf("failed to stat existing bundle %s: %w", bundle.irodsBundlePath, err) + return xerrors.Errorf("failed to stat existing bundle %q: %w", bundle.irodsBundlePath, err) } } @@ -1057,7 +1058,7 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error return irodsclient_types.NewFileNotFoundError(bundle.localBundlePath) } - return xerrors.Errorf("failed to stat %s: %w", bundle.localBundlePath, err) + return xerrors.Errorf("failed to stat %q: %w", bundle.localBundlePath, err) } if bundleEntry.Size == localBundleStat.Size() { @@ -1067,7 +1068,7 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error } if !haveExistingBundle { - logger.Debugf("uploading bundle %d to %s", bundle.index, bundle.irodsBundlePath) + logger.Debugf("uploading bundle %d to %q", bundle.index, bundle.irodsBundlePath) // determine how to download if manager.singleThreaded || manager.uploadThreadNum == 1 { @@ -1101,12 +1102,12 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error manager.progress(progressName, -1, totalFileSize, progress.UnitsBytes, true) } - return xerrors.Errorf("failed to upload bundle %d to %s: %w", bundle.index, bundle.irodsBundlePath, err) + return xerrors.Errorf("failed to upload bundle %d to %q: %w", bundle.index, bundle.irodsBundlePath, err) } - logger.Debugf("uploaded bundle %d to %s", bundle.index, bundle.irodsBundlePath) + logger.Debugf("uploaded bundle %d to %q", bundle.index, bundle.irodsBundlePath) } else { - logger.Debugf("skip uploading bundle %d to %s, file already exists", bundle.index, bundle.irodsBundlePath) + logger.Debugf("skip uploading bundle %d to %q, file already exists", bundle.index, bundle.irodsBundlePath) } // remove local bundle file @@ -1147,7 +1148,7 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error manager.progress(progressName, -1, totalFileSize, progress.UnitsBytes, true) } - return xerrors.Errorf("failed to create a dir %s to upload file %s in bundle %d to %s: %w", path.Dir(file.IRODSPath), file.LocalPath, bundle.index, file.IRODSPath, err) + return xerrors.Errorf("failed to create a directory %q to upload file %q in bundle %d to %q: %w", path.Dir(file.IRODSPath), file.LocalPath, bundle.index, file.IRODSPath, err) } } @@ -1159,10 +1160,10 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error manager.progress(progressName, -1, totalFileSize, progress.UnitsBytes, true) } - return xerrors.Errorf("failed to upload dir %s in bundle %d to %s: %w", file.LocalPath, bundle.index, file.IRODSPath, err) + return xerrors.Errorf("failed to upload a directory %q in bundle %d to %q: %w", file.LocalPath, bundle.index, file.IRODSPath, err) } - logger.Debugf("uploaded dir %s in bundle %d to %s", file.LocalPath, bundle.index, file.IRODSPath) + logger.Debugf("uploaded a directory %q in bundle %d to %q", file.LocalPath, bundle.index, file.IRODSPath) } else { // determine how to download if manager.singleThreaded || manager.uploadThreadNum == 1 { @@ -1196,14 +1197,14 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error manager.progress(progressName, -1, totalFileSize, progress.UnitsBytes, true) } - return xerrors.Errorf("failed to upload file %s in bundle %d to %s: %w", file.LocalPath, bundle.index, file.IRODSPath, err) + return xerrors.Errorf("failed to upload file %q in bundle %d to %q: %w", file.LocalPath, bundle.index, file.IRODSPath, err) } - logger.Debugf("uploaded file %s in bundle %d to %s", file.LocalPath, bundle.index, file.IRODSPath) + logger.Debugf("uploaded file %q in bundle %d to %q", file.LocalPath, bundle.index, file.IRODSPath) } } - logger.Debugf("uploaded files in bundle %d to %s", bundle.index, bundle.irodsBundlePath) + logger.Debugf("uploaded files in bundle %d to %q", bundle.index, bundle.irodsBundlePath) return nil } @@ -1214,7 +1215,7 @@ func (manager *BundleTransferManager) processBundleExtract(bundle *Bundle) error "function": "processBundleExtract", }) - logger.Debugf("extracting bundle %d at %s", bundle.index, bundle.irodsBundlePath) + logger.Debugf("extracting bundle %d at %q", bundle.index, bundle.irodsBundlePath) progressName := manager.getProgressName(bundle, BundleTaskNameExtract) @@ -1230,7 +1231,7 @@ func (manager *BundleTransferManager) processBundleExtract(bundle *Bundle) error manager.progress(progressName, totalFileNum, totalFileNum, progress.UnitsDefault, false) } - logger.Debugf("skip - extracting bundle %d at %s", bundle.index, bundle.irodsBundlePath) + logger.Debugf("skip - extracting bundle %d at %q", bundle.index, bundle.irodsBundlePath) return nil } @@ -1241,11 +1242,11 @@ func (manager *BundleTransferManager) processBundleExtract(bundle *Bundle) error } manager.filesystem.RemoveFile(bundle.irodsBundlePath, true) - return xerrors.Errorf("failed to extract bundle %d at %s to %s: %w", bundle.index, bundle.irodsBundlePath, manager.irodsDestPath, err) + return xerrors.Errorf("failed to extract bundle %d at %q to %q: %w", bundle.index, bundle.irodsBundlePath, manager.irodsDestPath, err) } // remove irods bundle file - logger.Debugf("removing bundle %d at %s", bundle.index, bundle.irodsBundlePath) + logger.Debugf("removing bundle %d at %q", bundle.index, bundle.irodsBundlePath) manager.filesystem.RemoveFile(bundle.irodsBundlePath, true) if manager.showProgress { @@ -1256,12 +1257,12 @@ func (manager *BundleTransferManager) processBundleExtract(bundle *Bundle) error bundle.Done() atomic.AddInt64(&manager.bundlesDoneCounter, 1) - logger.Debugf("extracted bundle %d at %s to %s", bundle.index, bundle.irodsBundlePath, manager.irodsDestPath) + logger.Debugf("extracted bundle %d at %q to %q", bundle.index, bundle.irodsBundlePath, manager.irodsDestPath) return nil } func (manager *BundleTransferManager) getProgressName(bundle *Bundle, taskName string) string { - return fmt.Sprintf("bundle %d - %s", bundle.index, taskName) + return fmt.Sprintf("bundle %d - %q", bundle.index, taskName) } func CleanUpOldLocalBundles(localTempDirPath string, force bool) { @@ -1271,11 +1272,11 @@ func CleanUpOldLocalBundles(localTempDirPath string, force bool) { "function": "CleanUpOldLocalBundles", }) - logger.Debugf("clearing local bundle files in %s", localTempDirPath) + logger.Debugf("clearing local bundle files in %q", localTempDirPath) entries, err := os.ReadDir(localTempDirPath) if err != nil { - logger.WithError(err).Warnf("failed to read local temp dir %s", localTempDirPath) + logger.WithError(err).Warnf("failed to read a local temp directory %q", localTempDirPath) return } @@ -1288,59 +1289,50 @@ func CleanUpOldLocalBundles(localTempDirPath string, force bool) { } } - if len(bundleEntries) == 0 { - return - } - - if force { - for _, entry := range bundleEntries { - logger.Debugf("deleting old local bundle %s", entry) - removeErr := os.Remove(entry) - if removeErr != nil { - logger.WithError(removeErr).Warnf("failed to remove old local bundle %s", entry) - } - } - return - } - deletedCount := 0 for _, entry := range bundleEntries { - // ask - del := InputYN(fmt.Sprintf("removing old local bundle file %s found. Delete?", entry)) - if del { - logger.Debugf("deleting old local bundle %s", entry) - + if force { + logger.Debugf("deleting old local bundle %q", entry) removeErr := os.Remove(entry) if removeErr != nil { - logger.WithError(removeErr).Warnf("failed to remove old local bundle %s", entry) - } else { - deletedCount++ + logger.WithError(removeErr).Warnf("failed to remove old local bundle %q", entry) + } + } else { + // ask + del := InputYN(fmt.Sprintf("removing old local bundle file %q found. Delete?", entry)) + if del { + logger.Debugf("deleting old local bundle %q", entry) + + removeErr := os.Remove(entry) + if removeErr != nil { + logger.WithError(removeErr).Warnf("failed to remove old local bundle %q", entry) + } else { + deletedCount++ + } } } } - Printf("deleted %d old local bundles in %s\n", deletedCount, localTempDirPath) - logger.Debugf("deleted %d old local bundles in %s", deletedCount, localTempDirPath) + Printf("deleted %d old local bundles in %q\n", deletedCount, localTempDirPath) + logger.Debugf("deleted %d old local bundles in %q", deletedCount, localTempDirPath) } -func CleanUpOldIRODSBundles(fs *irodsclient_fs.FileSystem, irodsTempDirPath string, removeDir bool, force bool) { +func CleanUpOldIRODSBundles(fs *irodsclient_fs.FileSystem, stagingPath string, removeDir bool, force bool) error { logger := log.WithFields(log.Fields{ "package": "commons", "struct": "BundleTransferManager", "function": "CleanUpOldIRODSBundles", }) - logger.Debugf("clearing old irods bundle files in %s", irodsTempDirPath) + logger.Debugf("cleaning up old irods bundle files in %q", stagingPath) - if !fs.ExistsDir(irodsTempDirPath) { - logger.Debugf("staging dir %s doesn't exist", irodsTempDirPath) - return + if !fs.ExistsDir(stagingPath) { + return xerrors.Errorf("staging dir %q does not exist", stagingPath) } - entries, err := fs.List(irodsTempDirPath) + entries, err := fs.List(stagingPath) if err != nil { - logger.WithError(err).Warnf("failed to listing staging dir %s", irodsTempDirPath) - return + return xerrors.Errorf("failed to list %q: %w", stagingPath, err) } deletedCount := 0 @@ -1348,10 +1340,10 @@ func CleanUpOldIRODSBundles(fs *irodsclient_fs.FileSystem, irodsTempDirPath stri // filter only bundle files if entry.Type == irodsclient_fs.FileEntry { if IsBundleFilename(entry.Name) { - logger.Debugf("deleting old irods bundle %s", entry.Path) + logger.Debugf("deleting old irods bundle %q", entry.Path) removeErr := fs.RemoveFile(entry.Path, force) if removeErr != nil { - logger.WithError(removeErr).Warnf("failed to remove old irods bundle %s", entry.Path) + return xerrors.Errorf("failed to remove bundle file %q: %w", entry.Path, removeErr) } else { deletedCount++ } @@ -1359,15 +1351,17 @@ func CleanUpOldIRODSBundles(fs *irodsclient_fs.FileSystem, irodsTempDirPath stri } } - Printf("deleted %d old irods bundles in %s\n", deletedCount, irodsTempDirPath) - logger.Debugf("deleted %d old irods bundles in %s", deletedCount, irodsTempDirPath) + Printf("deleted %d old irods bundles in %q\n", deletedCount, stagingPath) + logger.Debugf("deleted %d old irods bundles in %q", deletedCount, stagingPath) if removeDir { - if IsStagingDirInTargetPath(irodsTempDirPath) { - rmdirErr := fs.RemoveDir(irodsTempDirPath, true, force) + if IsStagingDirInTargetPath(stagingPath) { + rmdirErr := fs.RemoveDir(stagingPath, true, force) if rmdirErr != nil { - logger.WithError(rmdirErr).Warnf("failed to remove old irods bundle staging dir %s", irodsTempDirPath) + return xerrors.Errorf("failed to remove staging directory %q: %w", stagingPath, rmdirErr) } } } + + return nil } diff --git a/commons/commands.go b/commons/commands.go index 2dd4bcd..ca960e7 100644 --- a/commons/commands.go +++ b/commons/commands.go @@ -566,26 +566,26 @@ func LoadConfigFromFile(configPath string) error { configPath, err := ExpandHomeDir(configPath) if err != nil { - return xerrors.Errorf("failed to expand home dir for %s: %w", configPath, err) + return xerrors.Errorf("failed to expand home directory for %q: %w", configPath, err) } configPath, err = filepath.Abs(configPath) if err != nil { - return xerrors.Errorf("failed to compute absolute path for %s: %w", configPath, err) + return xerrors.Errorf("failed to compute absolute path for %q: %w", configPath, err) } - logger.Debugf("reading config file/dir - %s", configPath) + logger.Debugf("reading config path %q", configPath) // check if it is a file or a dir _, err = os.Stat(configPath) if err != nil { if os.IsNotExist(err) { return irodsclient_types.NewFileNotFoundError(configPath) } - return xerrors.Errorf("failed to stat %s: %w", configPath, err) + return xerrors.Errorf("failed to stat %q: %w", configPath, err) } if isYAMLFile(configPath) { - logger.Debugf("reading gocommands YAML config file - %s", configPath) + logger.Debugf("reading gocommands YAML config file %q", configPath) iCommandsEnvMgr, err := irodsclient_icommands.CreateIcommandsEnvironmentManager() if err != nil { @@ -594,7 +594,7 @@ func LoadConfigFromFile(configPath string) error { err = iCommandsEnvMgr.SetEnvironmentFilePath(configPath) if err != nil { - return xerrors.Errorf("failed to set environment file path %s: %w", configPath, err) + return xerrors.Errorf("failed to set environment file path %q: %w", configPath, err) } // read session @@ -602,7 +602,7 @@ func LoadConfigFromFile(configPath string) error { if util.ExistFile(sessionFilePath) { session, err := irodsclient_icommands.CreateICommandsEnvironmentFromFile(sessionFilePath) if err != nil { - return xerrors.Errorf("failed to create icommands environment from file %s: %w", sessionFilePath, err) + return xerrors.Errorf("failed to create icommands environment from file %q: %w", sessionFilePath, err) } iCommandsEnvMgr.Session = session @@ -611,7 +611,7 @@ func LoadConfigFromFile(configPath string) error { // load from YAML yjBytes, err := os.ReadFile(configPath) if err != nil { - return xerrors.Errorf("failed to read file %s: %w", configPath, err) + return xerrors.Errorf("failed to read file %q: %w", configPath, err) } defaultConfig := GetDefaultConfig() @@ -644,7 +644,7 @@ func LoadConfigFromFile(configPath string) error { configFilePath = filepath.Join(configPath, "irods_environment.json") } - logger.Debugf("reading icommands environment file - %s", configFilePath) + logger.Debugf("reading icommands environment file %q", configFilePath) iCommandsEnvMgr, err := irodsclient_icommands.CreateIcommandsEnvironmentManager() if err != nil { @@ -653,7 +653,7 @@ func LoadConfigFromFile(configPath string) error { err = iCommandsEnvMgr.SetEnvironmentFilePath(configFilePath) if err != nil { - return xerrors.Errorf("failed to set iCommands Environment file %s: %w", configFilePath, err) + return xerrors.Errorf("failed to set iCommands Environment file %q: %w", configFilePath, err) } err = iCommandsEnvMgr.Load(sessionID) @@ -744,89 +744,6 @@ func PrintAccount() error { return nil } -func PrintEnvironment() error { - envMgr := GetEnvironmentManager() - if envMgr == nil { - return xerrors.Errorf("environment is not set") - } - - t := table.NewWriter() - t.SetOutputMirror(os.Stdout) - - t.AppendRows([]table.Row{ - { - "iRODS Session Environment File", - envMgr.GetSessionFilePath(os.Getppid()), - }, - { - "iRODS Environment File", - envMgr.GetEnvironmentFilePath(), - }, - { - "iRODS Host", - envMgr.Environment.Host, - }, - { - "iRODS Port", - envMgr.Environment.Port, - }, - { - "iRODS Zone", - envMgr.Environment.Zone, - }, - { - "iRODS Username", - envMgr.Environment.Username, - }, - { - "iRODS Default Resource", - envMgr.Environment.DefaultResource, - }, - { - "iRODS Default Hash Scheme", - envMgr.Environment.DefaultHashScheme, - }, - { - "iRODS Authentication Scheme", - envMgr.Environment.AuthenticationScheme, - }, - { - "iRODS Client Server Negotiation", - envMgr.Environment.ClientServerNegotiation, - }, - { - "iRODS Client Server Policy", - envMgr.Environment.ClientServerPolicy, - }, - { - "iRODS SSL CA Certification File", - envMgr.Environment.SSLCACertificateFile, - }, - { - "iRODS SSL CA Certification Path", - envMgr.Environment.SSLCACertificatePath, - }, - { - "iRODS SSL Encryption Key Size", - envMgr.Environment.EncryptionKeySize, - }, - { - "iRODS SSL Encryption Key Algorithm", - envMgr.Environment.EncryptionAlgorithm, - }, - { - "iRODS SSL Encryption Salt Size", - envMgr.Environment.EncryptionSaltSize, - }, - { - "iRODS SSL Encryption Hash Rounds", - envMgr.Environment.EncryptionNumHashRounds, - }, - }, table.RowConfig{}) - t.Render() - return nil -} - // InputYN inputs Y or N // true for Y, false for N func InputYN(msg string) bool { diff --git a/commons/datetime.go b/commons/datetime.go index edc8bf8..ed010e2 100644 --- a/commons/datetime.go +++ b/commons/datetime.go @@ -32,7 +32,7 @@ func MakeDateTimeFromString(str string) (time.Time, error) { t, err := time.Parse(datetimeLayout, str) if err != nil { - return time.Time{}, xerrors.Errorf("failed to parse time %s: %w", str, err) + return time.Time{}, xerrors.Errorf("failed to parse time %q: %w", str, err) } return t, nil diff --git a/commons/encrypt_pgp.go b/commons/encrypt_pgp.go index 725e52d..778da7d 100644 --- a/commons/encrypt_pgp.go +++ b/commons/encrypt_pgp.go @@ -32,14 +32,14 @@ func DecryptFilenamePGP(filename string) string { func EncryptFilePGP(source string, target string, key []byte) error { sourceFileHandle, err := os.Open(source) if err != nil { - return xerrors.Errorf("failed to open file %s: %w", source, err) + return xerrors.Errorf("failed to open file %q: %w", source, err) } defer sourceFileHandle.Close() targetFileHandle, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { - return xerrors.Errorf("failed to create file %s: %w", target, err) + return xerrors.Errorf("failed to create file %q: %w", target, err) } defer targetFileHandle.Close() @@ -50,7 +50,7 @@ func EncryptFilePGP(source string, target string, key []byte) error { writeHandle, err := openpgp.SymmetricallyEncrypt(targetFileHandle, key, nil, encryptionConfig) if err != nil { - return xerrors.Errorf("failed to create a encrypt writer for %s: %w", target, err) + return xerrors.Errorf("failed to create a encrypt writer for %q: %w", target, err) } defer writeHandle.Close() @@ -66,14 +66,14 @@ func EncryptFilePGP(source string, target string, key []byte) error { func DecryptFilePGP(source string, target string, key []byte) error { sourceFileHandle, err := os.Open(source) if err != nil { - return xerrors.Errorf("failed to open file %s: %w", source, err) + return xerrors.Errorf("failed to open file %q: %w", source, err) } defer sourceFileHandle.Close() targetFileHandle, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { - return xerrors.Errorf("failed to create file %s: %w", target, err) + return xerrors.Errorf("failed to create file %q: %w", target, err) } defer targetFileHandle.Close() @@ -93,7 +93,7 @@ func DecryptFilePGP(source string, target string, key []byte) error { messageDetail, err := openpgp.ReadMessage(sourceFileHandle, nil, prompt, encryptionConfig) if err != nil { - return xerrors.Errorf("failed to decrypt for %s: %w", source, err) + return xerrors.Errorf("failed to decrypt for %q: %w", source, err) } _, err = io.Copy(targetFileHandle, messageDetail.UnverifiedBody) diff --git a/commons/encrypt_ssh.go b/commons/encrypt_ssh.go index 84b67e9..a392737 100644 --- a/commons/encrypt_ssh.go +++ b/commons/encrypt_ssh.go @@ -41,7 +41,7 @@ func GetDefaultPrivateKeyPath() string { func DecodePublicPrivateKey(keyPath string) (interface{}, error) { pemBytes, err := os.ReadFile(keyPath) if err != nil { - return nil, xerrors.Errorf("failed to read public/private key file %s: %w", keyPath, err) + return nil, xerrors.Errorf("failed to read public/private key file %q: %w", keyPath, err) } // is pem? @@ -55,26 +55,26 @@ func DecodePublicPrivateKey(keyPath string) (interface{}, error) { case "RSA PRIVATE KEY", "PRIVATE KEY", "EC PRIVATE KEY", "DSA PRIVATE KEY", "OPENSSH PRIVATE KEY": privateKey, err := ssh.ParseRawPrivateKey(pemBytes) if err != nil { - return nil, xerrors.Errorf("failed to parse private key file %s: %w", keyPath, err) + return nil, xerrors.Errorf("failed to parse private key file %q: %w", keyPath, err) } return privateKey, nil case "RSA PUBLIC KEY", "PUBLIC KEY", "EC PUBLIC KEY", "DSA PUBLIC KEY", "OPENSSH PUBLIC KEY": publicKey, err := ssh.ParsePublicKey(pemBytes) if err != nil { - return nil, xerrors.Errorf("failed to parse public key file %s: %w", keyPath, err) + return nil, xerrors.Errorf("failed to parse public key file %q: %w", keyPath, err) } return publicKey, nil default: - return nil, xerrors.Errorf("failed to parse public/private key file %s: %w", keyPath, err) + return nil, xerrors.Errorf("failed to parse public/private key file %q: %w", keyPath, err) } } // authorized key publicKey, _, _, _, err := ssh.ParseAuthorizedKey(pemBytes) if err != nil { - return nil, xerrors.Errorf("failed to parse public key file %s: %w", keyPath, err) + return nil, xerrors.Errorf("failed to parse public key file %q: %w", keyPath, err) } parsedCryptoKey, ok := publicKey.(ssh.CryptoPublicKey) diff --git a/commons/encrypt_ssh_rsa.go b/commons/encrypt_ssh_rsa.go index f6063b7..9e4146e 100644 --- a/commons/encrypt_ssh_rsa.go +++ b/commons/encrypt_ssh_rsa.go @@ -94,21 +94,21 @@ func DecryptFilenameSSH(filename string, privatekey *rsa.PrivateKey) (string, er func EncryptFileSSH(source string, target string, publickey *rsa.PublicKey) error { sourceFileHandle, err := os.Open(source) if err != nil { - return xerrors.Errorf("failed to open file %s: %w", source, err) + return xerrors.Errorf("failed to open file %q: %w", source, err) } defer sourceFileHandle.Close() targetFileHandle, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { - return xerrors.Errorf("failed to create file %s: %w", target, err) + return xerrors.Errorf("failed to create file %q: %w", target, err) } defer targetFileHandle.Close() stat, err := sourceFileHandle.Stat() if err != nil { - return xerrors.Errorf("failed to stat file %s: %w", source, err) + return xerrors.Errorf("failed to stat file %q: %w", source, err) } if stat.Size() == 0 { @@ -172,14 +172,14 @@ func EncryptFileSSH(source string, target string, publickey *rsa.PublicKey) erro func DecryptFileSSH(source string, target string, privatekey *rsa.PrivateKey) error { sourceFileHandle, err := os.Open(source) if err != nil { - return xerrors.Errorf("failed to open file %s: %w", source, err) + return xerrors.Errorf("failed to open file %q: %w", source, err) } defer sourceFileHandle.Close() targetFileHandle, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { - return xerrors.Errorf("failed to create file %s: %w", target, err) + return xerrors.Errorf("failed to create file %q: %w", target, err) } defer targetFileHandle.Close() diff --git a/commons/encrypt_winscp.go b/commons/encrypt_winscp.go index bbfbd13..7cd4e3b 100644 --- a/commons/encrypt_winscp.go +++ b/commons/encrypt_winscp.go @@ -89,21 +89,21 @@ func DecryptFilenameWinSCP(filename string, key []byte) (string, error) { func EncryptFileWinSCP(source string, target string, key []byte) error { sourceFileHandle, err := os.Open(source) if err != nil { - return xerrors.Errorf("failed to open file %s: %w", source, err) + return xerrors.Errorf("failed to open file %q: %w", source, err) } defer sourceFileHandle.Close() targetFileHandle, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { - return xerrors.Errorf("failed to create file %s: %w", target, err) + return xerrors.Errorf("failed to create file %q: %w", target, err) } defer targetFileHandle.Close() stat, err := sourceFileHandle.Stat() if err != nil { - return xerrors.Errorf("failed to stat file %s: %w", source, err) + return xerrors.Errorf("failed to stat file %q: %w", source, err) } if stat.Size() == 0 { @@ -143,14 +143,14 @@ func EncryptFileWinSCP(source string, target string, key []byte) error { func DecryptFileWinSCP(source string, target string, key []byte) error { sourceFileHandle, err := os.Open(source) if err != nil { - return xerrors.Errorf("failed to open file %s: %w", source, err) + return xerrors.Errorf("failed to open file %q: %w", source, err) } defer sourceFileHandle.Close() targetFileHandle, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { - return xerrors.Errorf("failed to create file %s: %w", target, err) + return xerrors.Errorf("failed to create file %q: %w", target, err) } defer targetFileHandle.Close() diff --git a/commons/errors.go b/commons/errors.go new file mode 100644 index 0000000..4a19ce7 --- /dev/null +++ b/commons/errors.go @@ -0,0 +1,68 @@ +package commons + +import ( + "errors" + "fmt" +) + +type NotDirError struct { + Path string +} + +func NewNotDirError(dest string) error { + return &NotDirError{ + Path: dest, + } +} + +// Error returns error message +func (err *NotDirError) Error() string { + return fmt.Sprintf("path %q is not a directory", err.Path) +} + +// Is tests type of error +func (err *NotDirError) Is(other error) bool { + _, ok := other.(*NotDirError) + return ok +} + +// ToString stringifies the object +func (err *NotDirError) ToString() string { + return fmt.Sprintf("NotDirError: %q", err.Path) +} + +// IsNotDirError evaluates if the given error is NotDirError +func IsNotDirError(err error) bool { + return errors.Is(err, &NotDirError{}) +} + +type NotFileError struct { + Path string +} + +func NewNotFileError(dest string) error { + return &NotFileError{ + Path: dest, + } +} + +// Error returns error message +func (err *NotFileError) Error() string { + return fmt.Sprintf("path %q is not a file", err.Path) +} + +// Is tests type of error +func (err *NotFileError) Is(other error) bool { + _, ok := other.(*NotFileError) + return ok +} + +// ToString stringifies the object +func (err *NotFileError) ToString() string { + return fmt.Sprintf("NotFileError: %q", err.Path) +} + +// IsNotFileError evaluates if the given error is NotFileError +func IsNotFileError(err error) bool { + return errors.Is(err, &NotFileError{}) +} diff --git a/commons/parallel.go b/commons/parallel.go index 4c3d4b6..d9eaf2b 100644 --- a/commons/parallel.go +++ b/commons/parallel.go @@ -277,7 +277,7 @@ func (manager *ParallelJobManager) Start() { logger.Debugf("# threads : %d, max %d", currentThreads, manager.maxThreads) go func(pjob *ParallelJob) { - logger.Debugf("Run job %d, %s", pjob.index, pjob.name) + logger.Debugf("Run job %d, %q", pjob.index, pjob.name) err := pjob.task(pjob) diff --git a/commons/path.go b/commons/path.go index 2076468..55d1c93 100644 --- a/commons/path.go +++ b/commons/path.go @@ -243,26 +243,13 @@ func commonPrefix(sep byte, paths ...string) string { } func GetCommonRootLocalDirPath(paths []string) (string, error) { - commonRootPath, err := GetCommonRootLocalDirPathForSync(paths) - if err != nil { - return "", err - } - - if commonRootPath == "/" { - return "/", nil - } - - return filepath.Dir(commonRootPath), nil -} - -func GetCommonRootLocalDirPathForSync(paths []string) (string, error) { absPaths := make([]string, len(paths)) // get abs paths for idx, path := range paths { absPath, err := filepath.Abs(path) if err != nil { - return "", xerrors.Errorf("failed to compute absolute path for %s: %w", path, err) + return "", xerrors.Errorf("failed to compute absolute path for %q: %w", path, err) } absPaths[idx] = absPath } @@ -270,18 +257,13 @@ func GetCommonRootLocalDirPathForSync(paths []string) (string, error) { // find shortest path commonRoot := commonPrefix(filepath.Separator, absPaths...) - realCommonRoot, err := ResolveSymlink(commonRoot) - if err != nil { - return "", xerrors.Errorf("failed to resolve symlink %s: %w", commonRoot, err) - } - - commonRootStat, err := os.Stat(realCommonRoot) + commonRootStat, err := os.Stat(commonRoot) if err != nil { if os.IsNotExist(err) { - return "", irodsclient_types.NewFileNotFoundError(realCommonRoot) + return "", irodsclient_types.NewFileNotFoundError(commonRoot) } - return "", xerrors.Errorf("failed to stat %s: %w", realCommonRoot, err) + return "", xerrors.Errorf("failed to stat %q: %w", commonRoot, err) } if commonRootStat.IsDir() { @@ -295,14 +277,14 @@ func ExpandHomeDir(p string) (string, error) { if p == "~" { homedir, err := os.UserHomeDir() if err != nil { - return "", xerrors.Errorf("failed to get user home dir: %w", err) + return "", xerrors.Errorf("failed to get user home directory: %w", err) } return homedir, nil } else if strings.HasPrefix(p, "~/") { homedir, err := os.UserHomeDir() if err != nil { - return "", xerrors.Errorf("failed to get user home dir: %w", err) + return "", xerrors.Errorf("failed to get user home directory: %w", err) } p = filepath.Join(homedir, p[2:]) @@ -341,20 +323,20 @@ func MarkPathMap(pathMap map[string]bool, p string) { func ResolveSymlink(p string) (string, error) { st, err := os.Lstat(p) if err != nil { - return "", xerrors.Errorf("failed to lstat path %s: %w", p, err) + return "", xerrors.Errorf("failed to lstat path %q: %w", p, err) } if st.Mode()&os.ModeSymlink == os.ModeSymlink { // symlink new_p, err := filepath.EvalSymlinks(p) if err != nil { - return "", xerrors.Errorf("failed to evaluate symlink path %s: %w", p, err) + return "", xerrors.Errorf("failed to evaluate symlink path %q: %w", p, err) } // follow recursively new_pp, err := ResolveSymlink(new_p) if err != nil { - return "", xerrors.Errorf("failed to evaluate symlink path %s: %w", new_p, err) + return "", xerrors.Errorf("failed to evaluate symlink path %q: %w", new_p, err) } return new_pp, nil diff --git a/commons/staging.go b/commons/staging.go index cd76b62..2e7a678 100644 --- a/commons/staging.go +++ b/commons/staging.go @@ -22,28 +22,28 @@ func IsBundleFilename(p string) bool { return false } -func ValidateStagingDir(fs *irodsclient_fs.FileSystem, targetPath string, stagingPath string) (bool, error) { +func IsSameResourceServer(fs *irodsclient_fs.FileSystem, path1 string, path2 string) (bool, error) { logger := log.WithFields(log.Fields{ "package": "commons", - "function": "ValidateStagingDir", + "function": "IsSameResourceServer", }) - stagingResourceServers, err := GetResourceServers(fs, stagingPath) + path1RS, err := GetResourceServers(fs, path1) if err != nil { - return false, xerrors.Errorf("failed to get resource servers for %s: %w", stagingPath, err) + return false, xerrors.Errorf("failed to get resource servers for %q: %w", path1, err) } - logger.Debugf("staging resource servers - %v", stagingResourceServers) + logger.Debugf("resource servers for path %q - %v", path1, path1RS) - targetResourceServers, err := GetResourceServers(fs, targetPath) + path2RS, err := GetResourceServers(fs, path2) if err != nil { - return false, xerrors.Errorf("failed to get resource servers for %s: %w", targetPath, err) + return false, xerrors.Errorf("failed to get resource servers for %q: %w", path2, err) } - logger.Debugf("target resource servers - %v", targetResourceServers) + logger.Debugf("staging resource servers for path %q - %v", path2, path2RS) - for _, stagingResourceServer := range stagingResourceServers { - for _, targetResourceServer := range targetResourceServers { + for _, stagingResourceServer := range path2RS { + for _, targetResourceServer := range path1RS { if stagingResourceServer == targetResourceServer { // same resource server return true, nil @@ -62,34 +62,34 @@ func IsStagingDirInTargetPath(stagingPath string) bool { return path.Base(stagingPath) == ".gocmd_staging" } -func CheckSafeStagingDir(stagingPath string) error { +func IsSafeStagingDir(stagingPath string) error { dirParts := strings.Split(stagingPath[1:], "/") dirDepth := len(dirParts) if dirDepth < 3 { // no - return xerrors.Errorf("staging path %s is not safe!", stagingPath) + return xerrors.Errorf("staging path %q is not safe!", stagingPath) } // zone/home/user OR zone/home/shared (public) if dirParts[0] != GetZone() { - return xerrors.Errorf("staging path %s is not safe, not in the correct zone", stagingPath) + return xerrors.Errorf("staging path %q is not safe, not in the correct zone", stagingPath) } if dirParts[1] != "home" { - return xerrors.Errorf("staging path %s is not safe", stagingPath) + return xerrors.Errorf("staging path %q is not safe", stagingPath) } if dirParts[2] == GetUsername() { if dirDepth <= 3 { // /zone/home/user - return xerrors.Errorf("staging path %s is not safe!", stagingPath) + return xerrors.Errorf("staging path %q is not safe!", stagingPath) } } else { // public or shared? if dirDepth <= 4 { // /zone/home/public/dataset1 - return xerrors.Errorf("staging path %s is not safe!", stagingPath) + return xerrors.Errorf("staging path %q is not safe!", stagingPath) } } @@ -111,7 +111,7 @@ func GetResourceServers(fs *irodsclient_fs.FileSystem, targetDir string) ([]stri if !fs.ExistsDir(targetDir) { err := fs.MakeDir(targetDir, true) if err != nil { - return nil, xerrors.Errorf("failed to make dir %s: %w", targetDir, err) + return nil, xerrors.Errorf("failed to make a directory %q: %w", targetDir, err) } dirCreated = true } @@ -121,7 +121,7 @@ func GetResourceServers(fs *irodsclient_fs.FileSystem, targetDir string) ([]stri filehandle, err := fs.CreateFile(testFilePath, "", "w+") if err != nil { - return nil, xerrors.Errorf("failed to create file %s: %w", testFilePath, err) + return nil, xerrors.Errorf("failed to create file %q: %w", testFilePath, err) } _, err = filehandle.Write([]byte("resource server test\n")) @@ -137,12 +137,12 @@ func GetResourceServers(fs *irodsclient_fs.FileSystem, targetDir string) ([]stri // data object collection, err := irodsclient_irodsfs.GetCollection(connection, targetDir) if err != nil { - return nil, xerrors.Errorf("failed to get collection %s: %w", targetDir, err) + return nil, xerrors.Errorf("failed to get collection %q: %w", targetDir, err) } entry, err := irodsclient_irodsfs.GetDataObject(connection, collection, path.Base(testFilePath)) if err != nil { - return nil, xerrors.Errorf("failed to get data-object %s: %w", testFilePath, err) + return nil, xerrors.Errorf("failed to get data-object %q: %w", testFilePath, err) } resourceServers := []string{} diff --git a/commons/tar.go b/commons/tar.go index cb3f905..9f17c39 100644 --- a/commons/tar.go +++ b/commons/tar.go @@ -30,23 +30,18 @@ func Tar(baseDir string, sources []string, target string, callback TrackerCallBa createdDirs := map[string]bool{} for _, source := range sources { - realSource, err := ResolveSymlink(source) - if err != nil { - return xerrors.Errorf("failed to resolve symlink %s: %w", source, err) - } - - sourceStat, err := os.Stat(realSource) + sourceStat, err := os.Stat(source) if err != nil { if os.IsNotExist(err) { - return irodsclient_types.NewFileNotFoundError(realSource) + return irodsclient_types.NewFileNotFoundError(source) } - return xerrors.Errorf("failed to stat %s: %w", realSource, err) + return xerrors.Errorf("failed to stat %q: %w", source, err) } rel, err := filepath.Rel(baseDir, source) if err != nil { - return xerrors.Errorf("failed to compute relative path %s to %s: %w", source, baseDir, err) + return xerrors.Errorf("failed to compute relative path %q to %q: %w", source, baseDir, err) } pdirs := GetParentLocalDirs(rel) @@ -76,18 +71,13 @@ func makeTar(entries []*TarEntry, target string, callback TrackerCallBack) error totalSize := int64(0) currentSize := int64(0) for _, entry := range entries { - realSource, err := ResolveSymlink(entry.source) - if err != nil { - return xerrors.Errorf("failed to resolve symlink %s: %w", entry.source, err) - } - - sourceStat, err := os.Stat(realSource) + sourceStat, err := os.Stat(entry.source) if err != nil { if os.IsNotExist(err) { return irodsclient_types.NewFileNotFoundError(entry.source) } - return xerrors.Errorf("failed to stat %s: %w", entry.source, err) + return xerrors.Errorf("failed to stat %q: %w", entry.source, err) } if !sourceStat.IsDir() { @@ -101,7 +91,7 @@ func makeTar(entries []*TarEntry, target string, callback TrackerCallBack) error tarfile, err := os.Create(target) if err != nil { - return xerrors.Errorf("failed to create file %s: %w", target, err) + return xerrors.Errorf("failed to create file %q: %w", target, err) } defer tarfile.Close() @@ -110,18 +100,13 @@ func makeTar(entries []*TarEntry, target string, callback TrackerCallBack) error defer tarWriter.Close() for _, entry := range entries { - realSource, err := ResolveSymlink(entry.source) - if err != nil { - return xerrors.Errorf("failed to resolve symlink %s: %w", entry.source, err) - } - - sourceStat, err := os.Stat(realSource) + sourceStat, err := os.Stat(entry.source) if err != nil { if os.IsNotExist(err) { return irodsclient_types.NewFileNotFoundError(entry.source) } - return xerrors.Errorf("failed to stat %s: %w", entry.source, err) + return xerrors.Errorf("failed to stat %q: %w", entry.source, err) } header, err := tar.FileInfoHeader(sourceStat, sourceStat.Name()) @@ -140,7 +125,7 @@ func makeTar(entries []*TarEntry, target string, callback TrackerCallBack) error // add file content file, err := os.Open(entry.source) if err != nil { - return xerrors.Errorf("failed to open tar file %s: %w", entry.source, err) + return xerrors.Errorf("failed to open tar file %q: %w", entry.source, err) } defer file.Close() diff --git a/commons/transfer_report.go b/commons/transfer_report.go index ec789ab..51c59f6 100644 --- a/commons/transfer_report.go +++ b/commons/transfer_report.go @@ -24,6 +24,10 @@ const ( TransferMethodPut TransferMethod = "PUT" // TransferMethodBput is for bput command TransferMethodBput TransferMethod = "BPUT" + // TransferMethodCopy is for cp command + TransferMethodCopy TransferMethod = "COPY" + // TransferMethodDelete is for delete command + TransferMethodDelete TransferMethod = "DELETE" // TransferMethodBputUnknown is for unknown command TransferMethodBputUnknown TransferMethod = "UNKNOWN" ) @@ -34,13 +38,13 @@ type TransferReportFile struct { StartAt time.Time `json:"start_time"` EndAt time.Time `json:"end_at"` - LocalPath string `json:"local_path"` - IrodsPath string `json:"irods_path"` + SourcePath string `json:"source_path"` + DestPath string `json:"dest_path"` ChecksumAlgorithm string `json:"checksum_algorithm"` - LocalSize int64 `json:"local_size"` - LocalChecksum string `json:"local_checksum"` - IrodsSize int64 `json:"irods_size"` - IrodsChecksum string `json:"irods_checksum"` + SourceSize int64 `json:"source_size"` + SourceChecksum string `json:"source_checksum"` + DestSize int64 `json:"dest_size"` + DestChecksum string `json:"dest_checksum"` Error error `json:"error,omitempty"` Notes []string `json:"notes"` // additional notes @@ -55,25 +59,58 @@ func GetTransferMethod(method string) TransferMethod { return TransferMethodPut case string(TransferMethodBput), "BULK_UPLOAD": return TransferMethodBput + case string(TransferMethodCopy), "CP": + return TransferMethodCopy + case string(TransferMethodDelete), "DEL": + return TransferMethodDelete default: return TransferMethodBputUnknown } } -func NewTransferReportFileFromTransferResult(result *irodsclient_fs.FileTransferResult, method TransferMethod, err error, notes []string) *TransferReportFile { - return &TransferReportFile{ - Method: method, - StartAt: result.StartTime, - EndAt: result.EndTime, - LocalPath: result.LocalPath, - LocalSize: result.LocalSize, - LocalChecksum: hex.EncodeToString(result.LocalCheckSum), - IrodsPath: result.IRODSPath, - IrodsSize: result.IRODSSize, - IrodsChecksum: hex.EncodeToString(result.IRODSCheckSum), - ChecksumAlgorithm: string(result.CheckSumAlgorithm), - Error: err, - Notes: notes, +func NewTransferReportFileFromTransferResult(result *irodsclient_fs.FileTransferResult, method TransferMethod, err error, notes []string) (*TransferReportFile, error) { + if method == TransferMethodGet { + // get + // source is irods, target is local + return &TransferReportFile{ + Method: method, + StartAt: result.StartTime, + EndAt: result.EndTime, + + SourcePath: result.IRODSPath, + SourceSize: result.IRODSSize, + SourceChecksum: hex.EncodeToString(result.IRODSCheckSum), + + DestPath: result.LocalPath, + DestSize: result.LocalSize, + DestChecksum: hex.EncodeToString(result.LocalCheckSum), + + ChecksumAlgorithm: string(result.CheckSumAlgorithm), + Error: err, + Notes: notes, + }, nil + } else if method == TransferMethodPut || method == TransferMethodBput { + // put + // source is local, target is irods + return &TransferReportFile{ + Method: method, + StartAt: result.StartTime, + EndAt: result.EndTime, + + SourcePath: result.LocalPath, + SourceSize: result.LocalSize, + SourceChecksum: hex.EncodeToString(result.LocalCheckSum), + + DestPath: result.IRODSPath, + DestSize: result.IRODSSize, + DestChecksum: hex.EncodeToString(result.IRODSCheckSum), + + ChecksumAlgorithm: string(result.CheckSumAlgorithm), + Error: err, + Notes: notes, + }, nil + } else { + return nil, xerrors.Errorf("unknown method %q", method) } } @@ -98,7 +135,7 @@ func NewTransferReportManager(report bool, reportPath string, reportToStdout boo // file fileWriter, err := os.Create(reportPath) if err != nil { - return nil, xerrors.Errorf("failed to create a report file %s: %w", reportPath, err) + return nil, xerrors.Errorf("failed to create a report file %q: %w", reportPath, err) } writer = fileWriter } @@ -142,7 +179,7 @@ func (manager *TransferReportManager) AddFile(file *TransferReportFile) error { lineOutput := "" if manager.reportToStdout { // line print - fmt.Printf("[%s]\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", file.Method, file.StartAt, file.EndAt, file.LocalPath, file.IrodsPath, file.LocalSize, file.IrodsSize, file.LocalChecksum, file.IrodsChecksum) + fmt.Printf("[%s]\t%s\t%s\t%s\t%d\t%s\t%s\t%d\t%s\n", file.Method, file.StartAt, file.EndAt, file.SourcePath, file.SourceSize, file.SourceChecksum, file.DestPath, file.DestSize, file.DestChecksum) } else { // json fileBytes, err := json.Marshal(file) @@ -163,6 +200,10 @@ func (manager *TransferReportManager) AddFile(file *TransferReportFile) error { // AddTransfer adds a new file transfer func (manager *TransferReportManager) AddTransfer(result *irodsclient_fs.FileTransferResult, method TransferMethod, err error, notes []string) error { - file := NewTransferReportFileFromTransferResult(result, method, err, notes) + file, err := NewTransferReportFileFromTransferResult(result, method, err, notes) + if err != nil { + return err + } + return manager.AddFile(file) } diff --git a/commons/unit.go b/commons/unit.go index df7e3b8..594d717 100644 --- a/commons/unit.go +++ b/commons/unit.go @@ -30,12 +30,12 @@ func ParseSize(size string) (int64, error) { case 'K', 'M', 'G', 'T': sizeNum, err = strconv.ParseInt(size[:len(size)-1], 10, 64) if err != nil { - return 0, xerrors.Errorf("failed to convert string '%s' to int: %w", size, err) + return 0, xerrors.Errorf("failed to convert string %q to int: %w", size, err) } default: sizeNum, err = strconv.ParseInt(size, 10, 64) if err != nil { - return 0, xerrors.Errorf("failed to convert string '%s' to int: %w", size, err) + return 0, xerrors.Errorf("failed to convert string %q to int: %w", size, err) } return sizeNum, nil } @@ -65,12 +65,12 @@ func ParseTime(t string) (int, error) { case 'S', 'M', 'H', 'D': tNum, err = strconv.ParseInt(t[:len(t)-1], 10, 64) if err != nil { - return 0, xerrors.Errorf("failed to convert string '%s' to int: %w", t, err) + return 0, xerrors.Errorf("failed to convert string %q to int: %w", t, err) } default: tNum, err = strconv.ParseInt(t, 10, 64) if err != nil { - return 0, xerrors.Errorf("failed to convert string '%s' to int: %w", t, err) + return 0, xerrors.Errorf("failed to convert string %q to int: %w", t, err) } return int(tNum), nil } From 7675366ab8ff8f0dbcb19e4f804f78673d35e533 Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Wed, 14 Aug 2024 12:03:49 -0700 Subject: [PATCH 05/16] refactor bput --- cmd/flag/bundle_transfer.go | 57 +-- cmd/subcmd/bclean.go | 19 +- cmd/subcmd/bput.go | 209 +++++++++-- cmd/subcmd/copy-sftp-id.go | 2 +- cmd/subcmd/cp.go | 10 +- cmd/subcmd/get.go | 14 +- cmd/subcmd/put.go | 14 +- cmd/subcmd/sync.go | 9 +- commons/bundle_transfer.go | 714 ++++++++++++++---------------------- 9 files changed, 491 insertions(+), 557 deletions(-) diff --git a/cmd/flag/bundle_transfer.go b/cmd/flag/bundle_transfer.go index d094334..d4ecd80 100644 --- a/cmd/flag/bundle_transfer.go +++ b/cmd/flag/bundle_transfer.go @@ -8,16 +8,11 @@ import ( "github.com/spf13/cobra" ) -type BundleTempFlagValues struct { - LocalTempPath string - IRODSTempPath string -} - -type BundleClearFlagValues struct { - Clear bool -} - -type BundleConfigFlagValues struct { +type BundleTransferFlagValues struct { + LocalTempPath string + IRODSTempPath string + ClearOld bool + MinFileNum int MaxFileNum int MaxFileSize int64 NoBulkRegistration bool @@ -25,37 +20,25 @@ type BundleConfigFlagValues struct { } var ( - bundleTempFlagValues BundleTempFlagValues - bundleClearFlagValues BundleClearFlagValues - bundleConfigFlagValues BundleConfigFlagValues + bundleTransferFlagValues BundleTransferFlagValues ) -func SetBundleTempFlags(command *cobra.Command) { - command.Flags().StringVar(&bundleTempFlagValues.LocalTempPath, "local_temp", os.TempDir(), "Specify local temp directory path to create bundle files") - command.Flags().StringVar(&bundleTempFlagValues.IRODSTempPath, "irods_temp", "", "Specify iRODS temp collection path to upload bundle files to") -} - -func GetBundleTempFlagValues() *BundleTempFlagValues { - return &bundleTempFlagValues -} - -func SetBundleClearFlags(command *cobra.Command) { - command.Flags().BoolVar(&bundleClearFlagValues.Clear, "clear", false, "Clear stale bundle files") -} - -func GetBundleClearFlagValues() *BundleClearFlagValues { - return &bundleClearFlagValues -} +func SetBundleTransferFlags(command *cobra.Command, displayTransferConfig bool) { + command.Flags().StringVar(&bundleTransferFlagValues.LocalTempPath, "local_temp", os.TempDir(), "Specify local temp directory path to create bundle files") + command.Flags().StringVar(&bundleTransferFlagValues.IRODSTempPath, "irods_temp", "", "Specify iRODS temp collection path to upload bundle files to") -func SetBundleConfigFlags(command *cobra.Command) { - command.Flags().IntVar(&bundleConfigFlagValues.MaxFileNum, "max_file_num", commons.MaxBundleFileNumDefault, "Specify max file number in a bundle file") - command.Flags().StringVar(&bundleConfigFlagValues.maxFileSizeInput, "max_file_size", strconv.FormatInt(commons.MaxBundleFileSizeDefault, 10), "Specify max file size of a bundle file") - command.Flags().BoolVar(&bundleConfigFlagValues.NoBulkRegistration, "no_bulk_reg", false, "Disable bulk registration") + if displayTransferConfig { + command.Flags().BoolVar(&bundleTransferFlagValues.ClearOld, "clear", false, "Clear stale bundle files") + command.Flags().IntVar(&bundleTransferFlagValues.MinFileNum, "min_file_num", commons.MinBundleFileNumDefault, "Specify min file number in a bundle file") + command.Flags().IntVar(&bundleTransferFlagValues.MaxFileNum, "max_file_num", commons.MaxBundleFileNumDefault, "Specify max file number in a bundle file") + command.Flags().StringVar(&bundleTransferFlagValues.maxFileSizeInput, "max_file_size", strconv.FormatInt(commons.MaxBundleFileSizeDefault, 10), "Specify max file size of a bundle file") + command.Flags().BoolVar(&bundleTransferFlagValues.NoBulkRegistration, "no_bulk_reg", false, "Disable bulk registration") + } } -func GetBundleConfigFlagValues() *BundleConfigFlagValues { - size, _ := commons.ParseSize(bundleConfigFlagValues.maxFileSizeInput) - bundleConfigFlagValues.MaxFileSize = size +func GetBundleTransferFlagValues() *BundleTransferFlagValues { + size, _ := commons.ParseSize(bundleTransferFlagValues.maxFileSizeInput) + bundleTransferFlagValues.MaxFileSize = size - return &bundleConfigFlagValues + return &bundleTransferFlagValues } diff --git a/cmd/subcmd/bclean.go b/cmd/subcmd/bclean.go index daf7eb8..6a54765 100644 --- a/cmd/subcmd/bclean.go +++ b/cmd/subcmd/bclean.go @@ -22,8 +22,7 @@ func AddBcleanCommand(rootCmd *cobra.Command) { // attach common flags flag.SetCommonFlags(bcleanCmd, false) - // attach bundle temp flags - flag.SetBundleTempFlags(bcleanCmd) + flag.SetBundleTransferFlags(bcleanCmd, false) flag.SetForceFlags(bcleanCmd, false) rootCmd.AddCommand(bcleanCmd) @@ -41,8 +40,8 @@ func processBcleanCommand(command *cobra.Command, args []string) error { type BcleanCommand struct { command *cobra.Command - forceFlagValues *flag.ForceFlagValues - bundleTempFlagValues *flag.BundleTempFlagValues + forceFlagValues *flag.ForceFlagValues + bundleTransferFlagValues *flag.BundleTransferFlagValues account *irodsclient_types.IRODSAccount filesystem *irodsclient_fs.FileSystem @@ -54,8 +53,8 @@ func NewBcleanCommand(command *cobra.Command, args []string) (*BcleanCommand, er bclean := &BcleanCommand{ command: command, - forceFlagValues: flag.GetForceFlagValues(), - bundleTempFlagValues: flag.GetBundleTempFlagValues(), + forceFlagValues: flag.GetForceFlagValues(), + bundleTransferFlagValues: flag.GetBundleTransferFlagValues(), } // path @@ -96,13 +95,13 @@ func (bclean *BcleanCommand) Process() error { // run // clear local - commons.CleanUpOldLocalBundles(bclean.bundleTempFlagValues.LocalTempPath, bclean.forceFlagValues.Force) + commons.CleanUpOldLocalBundles(bclean.bundleTransferFlagValues.LocalTempPath, bclean.forceFlagValues.Force) // clear remote - if len(bclean.bundleTempFlagValues.IRODSTempPath) > 0 { - logger.Debugf("clearing an irods temp directory %q", bclean.bundleTempFlagValues.IRODSTempPath) + if len(bclean.bundleTransferFlagValues.IRODSTempPath) > 0 { + logger.Debugf("clearing an irods temp directory %q", bclean.bundleTransferFlagValues.IRODSTempPath) - commons.CleanUpOldIRODSBundles(bclean.filesystem, bclean.bundleTempFlagValues.IRODSTempPath, true, bclean.forceFlagValues.Force) + commons.CleanUpOldIRODSBundles(bclean.filesystem, bclean.bundleTransferFlagValues.IRODSTempPath, true, bclean.forceFlagValues.Force) } else { userHome := commons.GetHomeDir() homeStagingDir := commons.GetDefaultStagingDir(userHome) diff --git a/cmd/subcmd/bput.go b/cmd/subcmd/bput.go index 5660e27..5da939f 100644 --- a/cmd/subcmd/bput.go +++ b/cmd/subcmd/bput.go @@ -1,6 +1,9 @@ package subcmd import ( + "bytes" + "encoding/hex" + "fmt" "io/fs" "os" "path" @@ -9,6 +12,7 @@ import ( irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" + irodsclient_util "github.com/cyverse/go-irodsclient/irods/util" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -29,9 +33,7 @@ func AddBputCommand(rootCmd *cobra.Command) { // attach common flags flag.SetCommonFlags(bputCmd, false) - flag.SetBundleTempFlags(bputCmd) - flag.SetBundleClearFlags(bputCmd) - flag.SetBundleConfigFlags(bputCmd) + flag.SetBundleTransferFlags(bputCmd, true) flag.SetParallelTransferFlags(bputCmd, true) flag.SetForceFlags(bputCmd, true) flag.SetProgressFlags(bputCmd) @@ -57,9 +59,7 @@ type BputCommand struct { command *cobra.Command forceFlagValues *flag.ForceFlagValues - bundleTempFlagValues *flag.BundleTempFlagValues - bundleClearFlagValues *flag.BundleClearFlagValues - bundleConfigFlagValues *flag.BundleConfigFlagValues + bundleTransferFlagValues *flag.BundleTransferFlagValues parallelTransferFlagValues *flag.ParallelTransferFlagValues progressFlagValues *flag.ProgressFlagValues retryFlagValues *flag.RetryFlagValues @@ -88,9 +88,7 @@ func NewBputCommand(command *cobra.Command, args []string) (*BputCommand, error) command: command, forceFlagValues: flag.GetForceFlagValues(), - bundleTempFlagValues: flag.GetBundleTempFlagValues(), - bundleClearFlagValues: flag.GetBundleClearFlagValues(), - bundleConfigFlagValues: flag.GetBundleConfigFlagValues(), + bundleTransferFlagValues: flag.GetBundleTransferFlagValues(), parallelTransferFlagValues: flag.GetParallelTransferFlagValues(), progressFlagValues: flag.GetProgressFlagValues(), retryFlagValues: flag.GetRetryFlagValues(), @@ -146,8 +144,8 @@ func (bput *BputCommand) Process() error { // clear local // delete local bundles before entering to retry - if bput.bundleClearFlagValues.Clear { - commons.CleanUpOldLocalBundles(bput.bundleTempFlagValues.LocalTempPath, true) + if bput.bundleTransferFlagValues.ClearOld { + commons.CleanUpOldLocalBundles(bput.bundleTransferFlagValues.LocalTempPath, true) } // handle retry @@ -188,7 +186,7 @@ func (bput *BputCommand) Process() error { } // clear old irods bundles - if bput.bundleClearFlagValues.Clear { + if bput.bundleTransferFlagValues.ClearOld { logger.Debugf("clearing an irods temp directory %q", stagingDirPath) err = commons.CleanUpOldIRODSBundles(bput.filesystem, stagingDirPath, false, true) if err != nil { @@ -209,7 +207,7 @@ func (bput *BputCommand) Process() error { } // bundle transfer manager - bput.bundleTransferManager = commons.NewBundleTransferManager(bput.filesystem, bput.targetPath, bundleRootPath, bput.bundleConfigFlagValues.MaxFileNum, bput.bundleConfigFlagValues.MaxFileSize, bput.parallelTransferFlagValues.SingleTread, bput.parallelTransferFlagValues.ThreadNumber, bput.parallelTransferFlagValues.RedirectToResource, bput.parallelTransferFlagValues.Icat, bput.bundleTempFlagValues.LocalTempPath, bput.bundleTempFlagValues.IRODSTempPath, bput.differentialTransferFlagValues.DifferentialTransfer, bput.differentialTransferFlagValues.NoHash, bput.bundleConfigFlagValues.NoBulkRegistration, bput.progressFlagValues.ShowProgress, bput.progressFlagValues.ShowFullPath) + bput.bundleTransferManager = commons.NewBundleTransferManager(bput.filesystem, bput.targetPath, bundleRootPath, bput.bundleTransferFlagValues.MinFileNum, bput.bundleTransferFlagValues.MaxFileNum, bput.bundleTransferFlagValues.MaxFileSize, bput.parallelTransferFlagValues.SingleTread, bput.parallelTransferFlagValues.ThreadNumber, bput.parallelTransferFlagValues.RedirectToResource, bput.parallelTransferFlagValues.Icat, bput.bundleTransferFlagValues.LocalTempPath, bput.bundleTransferFlagValues.IRODSTempPath, bput.bundleTransferFlagValues.NoBulkRegistration, bput.progressFlagValues.ShowProgress, bput.progressFlagValues.ShowFullPath) bput.bundleTransferManager.Start() // run @@ -281,8 +279,8 @@ func (bput *BputCommand) getStagingDir(targetPath string) (string, error) { zone := commons.GetZone() targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) - if len(bput.bundleTempFlagValues.IRODSTempPath) > 0 { - stagingPath := commons.MakeIRODSPath(cwd, home, zone, bput.bundleTempFlagValues.IRODSTempPath) + if len(bput.bundleTransferFlagValues.IRODSTempPath) > 0 { + stagingPath := commons.MakeIRODSPath(cwd, home, zone, bput.bundleTransferFlagValues.IRODSTempPath) createdDir := false tempEntry, err := bput.filesystem.Stat(stagingPath) @@ -320,14 +318,14 @@ func (bput *BputCommand) getStagingDir(targetPath string) (string, error) { ok, err := commons.IsSameResourceServer(bput.filesystem, targetPath, stagingPath) if err != nil { - logger.Debugf("failed to validate staging directory %q and target %q - %s", stagingPath, targetPath, err.Error()) + logger.WithError(err).Debugf("failed to validate staging directory %q and target %q", stagingPath, targetPath) if createdDir { bput.filesystem.RemoveDir(stagingPath, true, true) } stagingPath = commons.GetDefaultStagingDir(targetPath) - logger.Debugf("use default staging path %q for target %q - %s", stagingPath, targetPath, err.Error()) + logger.WithError(err).Debugf("use default staging path %q for target %q", stagingPath, targetPath) return stagingPath, nil } @@ -389,23 +387,184 @@ func (bput *BputCommand) bputOne(sourcePath string) error { return bput.putFile(sourceStat, sourcePath) } -func (bput *BputCommand) putFile(sourceStat fs.FileInfo, sourcePath string) error { - err := bput.bundleTransferManager.Schedule(sourcePath, false, sourceStat.Size(), sourceStat.ModTime().Local()) +func (bput *BputCommand) schedulePut(sourceStat fs.FileInfo, sourcePath string) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "BputCommand", + "function": "schedulePut", + }) + + err := bput.bundleTransferManager.Schedule(sourceStat, sourcePath) if err != nil { return xerrors.Errorf("failed to schedule a file %q: %w", sourcePath, err) } - //commons.MarkPathMap(bput.updatedPathMap, targetPath) + logger.Debugf("scheduled a file upload %q", sourcePath) return nil } +func (bput *BputCommand) putFile(sourceStat fs.FileInfo, sourcePath string) error { + logger := log.WithFields(log.Fields{ + "package": "subcmd", + "struct": "BputCommand", + "function": "putFile", + }) + + targetPath, err := bput.bundleTransferManager.GetTargetPath(sourcePath) + if err != nil { + return xerrors.Errorf("failed to get target path for source %q: %w", sourcePath, err) + } + + commons.MarkPathMap(bput.updatedPathMap, targetPath) + + targetEntry, err := bput.filesystem.Stat(targetPath) + if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // target does not exist + return bput.schedulePut(sourceStat, sourcePath) + } + + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + + // target exists + // target must be a file + if targetEntry.IsDir() { + return commons.NewNotFileError(targetPath) + } + + if bput.differentialTransferFlagValues.DifferentialTransfer { + if bput.differentialTransferFlagValues.NoHash { + if targetEntry.Size == sourceStat.Size() { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + SourceSize: sourceStat.Size(), + + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"differential", "no_hash", "same file size", "skip"}, + } + + bput.transferReportManager.AddFile(reportFile) + + commons.Printf("skip uploading a file %q to %q. The file already exists!\n", sourcePath, targetPath) + logger.Debugf("skip uploading a file %q to %q. The file already exists!", sourcePath, targetPath) + return nil + } + } else { + if targetEntry.Size == sourceStat.Size() { + // compare hash + if len(targetEntry.CheckSum) > 0 { + localChecksum, err := irodsclient_util.HashLocalFile(sourcePath, string(targetEntry.CheckSumAlgorithm)) + if err != nil { + return xerrors.Errorf("failed to get hash %q: %w", sourcePath, err) + } + + if bytes.Equal(localChecksum, targetEntry.CheckSum) { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + SourceSize: sourceStat.Size(), + SourceChecksum: hex.EncodeToString(localChecksum), + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"differential", "same checksum", "skip"}, + } + + bput.transferReportManager.AddFile(reportFile) + + commons.Printf("skip uploading a file %q to %q. The file with the same hash already exists!\n", sourcePath, targetPath) + logger.Debugf("skip uploading a file %q to %q. The file with the same hash already exists!", sourcePath, targetPath) + return nil + } + } + } + } + } else { + if !bput.forceFlagValues.Force { + // ask + overwrite := commons.InputYN(fmt.Sprintf("file %q already exists. Overwrite?", targetPath)) + if !overwrite { + // skip + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + SourceSize: sourceStat.Size(), + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"no_overwrite", "skip"}, + } + + bput.transferReportManager.AddFile(reportFile) + + commons.Printf("skip uploading a file %q to %q. The data object already exists!\n", sourcePath, targetPath) + logger.Debugf("skip uploading a file %q to %q. The data object already exists!", sourcePath, targetPath) + return nil + } + } + } + + // schedule + return bput.schedulePut(sourceStat, sourcePath) +} + func (bput *BputCommand) putDir(sourceStat fs.FileInfo, sourcePath string) error { - err := bput.bundleTransferManager.Schedule(sourcePath, true, 0, sourceStat.ModTime().Local()) + targetPath, err := bput.bundleTransferManager.GetTargetPath(sourcePath) if err != nil { - return xerrors.Errorf("failed to schedule a directory %q: %w", sourcePath, err) + return xerrors.Errorf("failed to get target path for source %q: %w", sourcePath, err) } + commons.MarkPathMap(bput.updatedPathMap, targetPath) + + targetEntry, err := bput.filesystem.Stat(targetPath) + if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // target does not exist + err = bput.filesystem.MakeDir(targetPath, true) + if err != nil { + return xerrors.Errorf("failed to make a collection %q: %w", targetPath, err) + } + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + DestPath: targetPath, + Notes: []string{"directory"}, + } + + bput.transferReportManager.AddFile(reportFile) + } else { + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) + } + } else { + // target exists + if !targetEntry.IsDir() { + return commons.NewNotDirError(targetPath) + } + } + + // get entries entries, err := os.ReadDir(sourcePath) if err != nil { return xerrors.Errorf("failed to read a directory %q: %w", sourcePath, err) @@ -429,21 +588,15 @@ func (bput *BputCommand) putDir(sourceStat fs.FileInfo, sourcePath string) error if err != nil { return err } - - //commons.MarkPathMap(bput.updatedPathMap, newEntryPath) } else { // file err = bput.putFile(entryStat, entryPath) if err != nil { return err } - - //commons.MarkPathMap(bput.updatedPathMap, newEntryPath) } } - //commons.MarkPathMap(put.updatedPathMap, targetPath) - return nil } diff --git a/cmd/subcmd/copy-sftp-id.go b/cmd/subcmd/copy-sftp-id.go index 20859d2..d572a17 100644 --- a/cmd/subcmd/copy-sftp-id.go +++ b/cmd/subcmd/copy-sftp-id.go @@ -236,7 +236,7 @@ func (copy *CopySftpIdCommand) updateAuthorizedKeys(identityFiles []string, auth authorizedKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(keyLine)) if err != nil { // skip - log.Debugf("failed to parse a authorized key line - %s", err.Error()) + log.WithError(err).Debugf("failed to parse a authorized key line") continue } diff --git a/cmd/subcmd/cp.go b/cmd/subcmd/cp.go index 87fc213..1def183 100644 --- a/cmd/subcmd/cp.go +++ b/cmd/subcmd/cp.go @@ -287,8 +287,6 @@ func (cp *CpCommand) scheduleCopy(sourceEntry *irodsclient_fs.Entry, targetPath return xerrors.Errorf("failed to schedule copy %q to %q: %w", sourceEntry.Path, targetPath, err) } - commons.MarkPathMap(cp.updatedPathMap, targetPath) - logger.Debugf("scheduled a data object copy %q to %q", sourceEntry.Path, targetPath) return nil @@ -301,6 +299,8 @@ func (cp *CpCommand) copyFile(sourceEntry *irodsclient_fs.Entry, targetPath stri "function": "copyFile", }) + commons.MarkPathMap(cp.updatedPathMap, targetPath) + targetEntry, err := cp.filesystem.Stat(targetPath) if err != nil { if irodsclient_types.IsFileNotFoundError(err) { @@ -404,6 +404,8 @@ func (cp *CpCommand) copyFile(sourceEntry *irodsclient_fs.Entry, targetPath stri } func (cp *CpCommand) copyDir(sourceEntry *irodsclient_fs.Entry, targetPath string) error { + commons.MarkPathMap(cp.updatedPathMap, targetPath) + targetEntry, err := cp.filesystem.Stat(targetPath) if err != nil { if irodsclient_types.IsFileNotFoundError(err) { @@ -457,12 +459,8 @@ func (cp *CpCommand) copyDir(sourceEntry *irodsclient_fs.Entry, targetPath strin return err } } - - commons.MarkPathMap(cp.updatedPathMap, newEntryPath) } - commons.MarkPathMap(cp.updatedPathMap, targetPath) - return nil } diff --git a/cmd/subcmd/get.go b/cmd/subcmd/get.go index d8c50c0..62bc7ee 100644 --- a/cmd/subcmd/get.go +++ b/cmd/subcmd/get.go @@ -426,8 +426,6 @@ func (get *GetCommand) scheduleGet(sourceEntry *irodsclient_fs.Entry, tempPath s return xerrors.Errorf("failed to schedule download %q to %q: %w", sourceEntry.Path, targetPath, err) } - commons.MarkPathMap(get.updatedPathMap, targetPath) - logger.Debugf("scheduled a data object download %q to %q", sourceEntry.Path, targetPath) return nil @@ -440,6 +438,8 @@ func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath strin "function": "getFile", }) + commons.MarkPathMap(get.updatedPathMap, targetPath) + targetStat, err := os.Stat(targetPath) if err != nil { if os.IsNotExist(err) { @@ -560,6 +560,8 @@ func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath strin } func (get *GetCommand) getDir(sourceEntry *irodsclient_fs.Entry, targetPath string, parentDecryption bool) error { + commons.MarkPathMap(get.updatedPathMap, targetPath) + targetStat, err := os.Stat(targetPath) if err != nil { if os.IsNotExist(err) { @@ -609,8 +611,6 @@ func (get *GetCommand) getDir(sourceEntry *irodsclient_fs.Entry, targetPath stri if err != nil { return err } - - commons.MarkPathMap(get.updatedPathMap, newEntryPath) } else { // file if requireDecryption { @@ -624,21 +624,15 @@ func (get *GetCommand) getDir(sourceEntry *irodsclient_fs.Entry, targetPath stri if err != nil { return err } - - commons.MarkPathMap(get.updatedPathMap, newTargetPath) } else { err = get.getFile(entry, "", newEntryPath, requireDecryption) if err != nil { return err } - - commons.MarkPathMap(get.updatedPathMap, newEntryPath) } } } - commons.MarkPathMap(get.updatedPathMap, targetPath) - return nil } diff --git a/cmd/subcmd/put.go b/cmd/subcmd/put.go index dc536c7..a0a6ec6 100644 --- a/cmd/subcmd/put.go +++ b/cmd/subcmd/put.go @@ -436,8 +436,6 @@ func (put *PutCommand) schedulePut(sourceStat fs.FileInfo, sourcePath string, te return xerrors.Errorf("failed to schedule upload %q to %q: %w", sourcePath, targetPath, err) } - commons.MarkPathMap(put.updatedPathMap, targetPath) - logger.Debugf("scheduled a file upload %q to %q", sourcePath, targetPath) return nil @@ -450,6 +448,8 @@ func (put *PutCommand) putFile(sourceStat fs.FileInfo, sourcePath string, tempPa "function": "putFile", }) + commons.MarkPathMap(put.updatedPathMap, targetPath) + targetEntry, err := put.filesystem.Stat(targetPath) if err != nil { if irodsclient_types.IsFileNotFoundError(err) { @@ -560,6 +560,8 @@ func (put *PutCommand) putFile(sourceStat fs.FileInfo, sourcePath string, tempPa } func (put *PutCommand) putDir(sourceStat fs.FileInfo, sourcePath string, targetPath string, parentEncryption bool, parentEncryptionMode commons.EncryptionMode) error { + commons.MarkPathMap(put.updatedPathMap, targetPath) + targetEntry, err := put.filesystem.Stat(targetPath) if err != nil { if irodsclient_types.IsFileNotFoundError(err) { @@ -619,8 +621,6 @@ func (put *PutCommand) putDir(sourceStat fs.FileInfo, sourcePath string, targetP if err != nil { return err } - - commons.MarkPathMap(put.updatedPathMap, newEntryPath) } else { // file if requireEncryption { @@ -634,21 +634,15 @@ func (put *PutCommand) putDir(sourceStat fs.FileInfo, sourcePath string, targetP if err != nil { return err } - - commons.MarkPathMap(put.updatedPathMap, newTargetPath) } else { err = put.putFile(entryStat, entryPath, "", newEntryPath, requireEncryption, encryptionMode) if err != nil { return err } - - commons.MarkPathMap(put.updatedPathMap, newEntryPath) } } } - commons.MarkPathMap(put.updatedPathMap, targetPath) - return nil } diff --git a/cmd/subcmd/sync.go b/cmd/subcmd/sync.go index b90577e..f7a1751 100644 --- a/cmd/subcmd/sync.go +++ b/cmd/subcmd/sync.go @@ -4,8 +4,6 @@ import ( "os" "strings" - irodsclient_fs "github.com/cyverse/go-irodsclient/fs" - irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" @@ -26,9 +24,7 @@ func AddSyncCommand(rootCmd *cobra.Command) { // attach common flags flag.SetCommonFlags(syncCmd, false) - flag.SetBundleTempFlags(syncCmd) - flag.SetBundleClearFlags(syncCmd) - flag.SetBundleConfigFlags(syncCmd) + flag.SetBundleTransferFlags(syncCmd, true) flag.SetParallelTransferFlags(syncCmd, true) flag.SetForceFlags(syncCmd, true) flag.SetProgressFlags(syncCmd) @@ -54,9 +50,6 @@ type SyncCommand struct { retryFlagValues *flag.RetryFlagValues - account *irodsclient_types.IRODSAccount - filesystem *irodsclient_fs.FileSystem - sourcePaths []string targetPath string } diff --git a/commons/bundle_transfer.go b/commons/bundle_transfer.go index d72d8ff..03bb3a7 100644 --- a/commons/bundle_transfer.go +++ b/commons/bundle_transfer.go @@ -1,15 +1,14 @@ package commons import ( - "bytes" "encoding/hex" "fmt" + "io/fs" "os" "path" "path/filepath" "sync" "sync/atomic" - "time" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" @@ -43,29 +42,29 @@ type BundleEntry struct { type Bundle struct { manager *BundleTransferManager - index int64 - entries []*BundleEntry - size int64 - localBundlePath string - irodsBundlePath string - lastError error - lastErrorTaskName string + Index int64 + Entries []*BundleEntry + Size int64 + LocalBundlePath string + IRODSBundlePath string + LastError error + LastErrorTaskName string - done bool + Completed bool } func newBundle(manager *BundleTransferManager) (*Bundle, error) { bundle := &Bundle{ manager: manager, - index: manager.getNextBundleIndex(), - entries: []*BundleEntry{}, - size: 0, - localBundlePath: "", - irodsBundlePath: "", - lastError: nil, - lastErrorTaskName: "", - - done: false, + Index: manager.getNextBundleIndex(), + Entries: []*BundleEntry{}, + Size: 0, + LocalBundlePath: "", + IRODSBundlePath: "", + LastError: nil, + LastErrorTaskName: "", + + Completed: false, } err := bundle.updateBundlePath() @@ -76,8 +75,12 @@ func newBundle(manager *BundleTransferManager) (*Bundle, error) { return bundle, nil } +func (bundle *Bundle) GetManager() *BundleTransferManager { + return bundle.manager +} + func (bundle *Bundle) GetEntries() []*BundleEntry { - return bundle.entries + return bundle.Entries } func (bundle *Bundle) GetBundleFilename() (string, error) { @@ -85,7 +88,7 @@ func (bundle *Bundle) GetBundleFilename() (string, error) { entryStrs = append(entryStrs, "empty_bundle") - for _, entry := range bundle.entries { + for _, entry := range bundle.Entries { entryStrs = append(entryStrs, entry.LocalPath) } @@ -99,45 +102,24 @@ func (bundle *Bundle) GetBundleFilename() (string, error) { return GetBundleFilename(hexhash), nil } -func (bundle *Bundle) AddFile(localPath string, size int64) error { - irodsPath, err := bundle.manager.getTargetPath(localPath) +func (bundle *Bundle) Add(sourceStat fs.FileInfo, sourcePath string) error { + irodsPath, err := bundle.manager.GetTargetPath(sourcePath) if err != nil { - return xerrors.Errorf("failed to get target path for %q: %w", localPath, err) + return xerrors.Errorf("failed to get target path for %q: %w", sourcePath, err) } e := &BundleEntry{ - LocalPath: localPath, + LocalPath: sourcePath, IRODSPath: irodsPath, - Size: size, - Dir: false, - } - - bundle.entries = append(bundle.entries, e) - bundle.size += size - - err = bundle.updateBundlePath() - if err != nil { - return err - } - - return nil -} - -func (bundle *Bundle) AddDir(localPath string) error { - irodsPath, err := bundle.manager.getTargetPath(localPath) - if err != nil { - return xerrors.Errorf("failed to get target path for %q: %w", localPath, err) + Size: sourceStat.Size(), + Dir: sourceStat.IsDir(), } - e := &BundleEntry{ - LocalPath: localPath, - IRODSPath: irodsPath, - Size: 0, - Dir: true, + bundle.Entries = append(bundle.Entries, e) + if !sourceStat.IsDir() { + bundle.Size += sourceStat.Size() } - bundle.entries = append(bundle.entries, e) - err = bundle.updateBundlePath() if err != nil { return err @@ -152,21 +134,21 @@ func (bundle *Bundle) updateBundlePath() error { return xerrors.Errorf("failed to get bundle filename: %w", err) } - bundle.localBundlePath = filepath.Join(bundle.manager.localTempDirPath, filename) - bundle.irodsBundlePath = filepath.Join(bundle.manager.irodsTempDirPath, filename) + bundle.LocalBundlePath = filepath.Join(bundle.manager.localTempDirPath, filename) + bundle.IRODSBundlePath = filepath.Join(bundle.manager.irodsTempDirPath, filename) return nil } func (bundle *Bundle) isFull() bool { - return bundle.size >= bundle.manager.maxBundleFileSize || len(bundle.entries) >= bundle.manager.maxBundleFileNum + return bundle.Size >= bundle.manager.maxBundleFileSize || len(bundle.Entries) >= bundle.manager.maxBundleFileNum } -func (bundle *Bundle) requireTar() bool { - return len(bundle.entries) >= MinBundleFileNumDefault +func (bundle *Bundle) RequireTar() bool { + return len(bundle.Entries) >= bundle.manager.minBundleFileNum } -func (bundle *Bundle) Done() { - bundle.done = true +func (bundle *Bundle) SetCompleted() { + bundle.Completed = true } type BundleTransferManager struct { @@ -176,8 +158,8 @@ type BundleTransferManager struct { nextBundleIndex int64 pendingBundles chan *Bundle bundles []*Bundle - inputPathMap map[string]bool bundleRootPath string + minBundleFileNum int maxBundleFileNum int maxBundleFileSize int64 singleThreaded bool @@ -186,8 +168,6 @@ type BundleTransferManager struct { useIcat bool localTempDirPath string irodsTempDirPath string - differentFilesOnly bool - noHashForComparison bool noBulkRegistration bool showProgress bool showFullPath bool @@ -205,7 +185,7 @@ type BundleTransferManager struct { } // NewBundleTransferManager creates a new BundleTransferManager -func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, irodsDestPath string, bundleRootPath string, maxBundleFileNum int, maxBundleFileSize int64, singleThreaded bool, uploadThreadNum int, redirectToResource bool, useIcat bool, localTempDirPath string, irodsTempDirPath string, diff bool, noHash bool, noBulkReg bool, showProgress bool, showFullPath bool) *BundleTransferManager { +func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, irodsDestPath string, bundleRootPath string, minBundleFileNum int, maxBundleFileNum int, maxBundleFileSize int64, singleThreaded bool, uploadThreadNum int, redirectToResource bool, useIcat bool, localTempDirPath string, irodsTempDirPath string, noBulkReg bool, showProgress bool, showFullPath bool) *BundleTransferManager { cwd := GetCWD() home := GetHomeDir() zone := GetZone() @@ -218,8 +198,8 @@ func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, irodsDestPath strin nextBundleIndex: 0, pendingBundles: make(chan *Bundle, 100), bundles: []*Bundle{}, - inputPathMap: map[string]bool{}, bundleRootPath: "/", + minBundleFileNum: minBundleFileNum, maxBundleFileNum: maxBundleFileNum, maxBundleFileSize: maxBundleFileSize, singleThreaded: singleThreaded, @@ -228,8 +208,6 @@ func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, irodsDestPath strin useIcat: useIcat, localTempDirPath: localTempDirPath, irodsTempDirPath: irodsTempDirPath, - differentFilesOnly: diff, - noHashForComparison: noHash, noBulkRegistration: noBulkReg, showProgress: showProgress, showFullPath: showFullPath, @@ -245,6 +223,14 @@ func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, irodsDestPath strin bundlesDoneCounter: 0, } + if manager.maxBundleFileNum <= 0 { + manager.maxBundleFileNum = MaxBundleFileNumDefault + } + + if manager.minBundleFileNum <= 0 { + manager.minBundleFileNum = MinBundleFileNumDefault + } + if manager.uploadThreadNum > UploadTreadNumMax { manager.uploadThreadNum = UploadTreadNumMax } @@ -270,7 +256,7 @@ func (manager *BundleTransferManager) progress(name string, processed int64, tot } } -func (manager *BundleTransferManager) getTargetPath(localPath string) (string, error) { +func (manager *BundleTransferManager) GetTargetPath(localPath string) (string, error) { relPath, err := filepath.Rel(manager.bundleRootPath, localPath) if err != nil { return "", xerrors.Errorf("failed to compute relative path %q to %q: %w", localPath, manager.bundleRootPath, err) @@ -279,7 +265,7 @@ func (manager *BundleTransferManager) getTargetPath(localPath string) (string, e return path.Join(manager.irodsDestPath, filepath.ToSlash(relPath)), nil } -func (manager *BundleTransferManager) Schedule(source string, dir bool, size int64, lastModTime time.Time) error { +func (manager *BundleTransferManager) Schedule(sourceStat fs.FileInfo, sourcePath string) error { logger := log.WithFields(log.Fields{ "package": "commons", "struct": "BundleTransferManager", @@ -314,88 +300,17 @@ func (manager *BundleTransferManager) Schedule(source string, dir bool, size int // add new bundle, err := newBundle(manager) if err != nil { - return xerrors.Errorf("failed to create a new bundle for %q: %w", source, err) + return xerrors.Errorf("failed to create a new bundle for %q: %w", sourcePath, err) } manager.currentBundle = bundle - logger.Debugf("assigned a new bundle %d", manager.currentBundle.index) + logger.Debugf("assigned a new bundle %d", manager.currentBundle.Index) } defer manager.mutex.Unlock() - targePath, err := manager.getTargetPath(source) - if err != nil { - return xerrors.Errorf("failed to get target path for %q: %w", source, err) - } - MarkPathMap(manager.inputPathMap, targePath) - - if manager.differentFilesOnly { - logger.Debugf("checking if target %q for source %q exists", targePath, source) - - if dir { - // handle dir - exist := manager.filesystem.ExistsDir(targePath) - if exist { - Printf("skip adding a directory %q to the bundle. The dir already exists!\n", source) - logger.Debugf("skip adding a directory %q to the bundle. The dir already exists!", source) - return nil - } - - logger.Debugf("adding a directory %q to the bundle as it doesn't exist", source) - } else { - exist := manager.filesystem.ExistsFile(targePath) - if exist { - targetEntry, err := manager.filesystem.Stat(targePath) - if err != nil { - return xerrors.Errorf("failed to stat %q: %w", targePath, err) - } - - if manager.noHashForComparison { - if targetEntry.Size == size { - Printf("skip adding a file %q to the bundle. The file already exists!\n", source) - logger.Debugf("skip adding a file %q to the bundle. The file already exists!", source) - return nil - } - - logger.Debugf("adding a file %q to the bundle as it has different size %d != %d", source, targetEntry.Size, size) - } else { - if targetEntry.Size == size { - if len(targetEntry.CheckSum) > 0 { - // compare hash - hash, err := irodsclient_util.HashLocalFile(source, string(targetEntry.CheckSumAlgorithm)) - if err != nil { - return xerrors.Errorf("failed to get hash %q: %w", source, err) - } - - if bytes.Equal(hash, targetEntry.CheckSum) { - Printf("skip adding a file %q to the bundle. The file with the same hash already exists!\n", source) - logger.Debugf("skip adding a file %q to the bundle. The file with the same hash already exists!", source) - return nil - } - - logger.Debugf("adding a file %q to the bundle as it has different hash, %q vs %q (alg %q)", source, hash, targetEntry.CheckSum, targetEntry.CheckSumAlgorithm) - } else { - logger.Debugf("adding a file %q to the bundle as the file in iRODS doesn't have hash yet", source) - } - } else { - logger.Debugf("adding a file %q to the bundle as it has different size %d != %d", source, targetEntry.Size, size) - } - } - } else { - logger.Debugf("adding a file %q to the bundle as it doesn't exist", source) - } - } - } - - if dir { - manager.currentBundle.AddDir(source) - logger.Debugf("> scheduled a local file bundle-upload %q", source) - return nil - } - - manager.currentBundle.AddFile(source, size) - logger.Debugf("> scheduled a local file bundle-upload %q", source) - return nil + logger.Debugf("scheduling a local file/directory bundle-upload %q", sourcePath) + return manager.currentBundle.Add(sourceStat, sourcePath) } func (manager *BundleTransferManager) DoneScheduling() { @@ -417,10 +332,6 @@ func (manager *BundleTransferManager) GetBundles() []*Bundle { return manager.bundles } -func (manager *BundleTransferManager) GetInputPathMap() map[string]bool { - return manager.inputPathMap -} - func (manager *BundleTransferManager) Wait() error { logger := log.WithFields(log.Fields{ "package": "commons", @@ -458,59 +369,8 @@ func (manager *BundleTransferManager) CleanUpBundles() { logger.Debugf("clearing bundle files in %q", manager.irodsTempDirPath) - // if the staging dir is not in target path - entries, err := manager.filesystem.List(manager.irodsTempDirPath) - if err != nil { - logger.WithError(err).Warnf("failed to listing a staging directory %q", manager.irodsTempDirPath) - return - } - - if len(entries) == 0 { - // empty - // remove the dir - err := manager.filesystem.RemoveDir(manager.irodsTempDirPath, true, true) - if err != nil { - logger.WithError(err).Warnf("failed to remove staging directory %q", manager.irodsTempDirPath) - return - } - return - } - - // has some files in it yet - // ask - deleted := 0 - for _, entry := range entries { - del := InputYN(fmt.Sprintf("removing old bundle file %q found. Delete?", entry.Path)) - if del { - logger.Debugf("deleting old bundle file %q", entry.Path) - - removeErr := os.Remove(entry.Path) - if removeErr != nil { - logger.WithError(removeErr).Warnf("failed to remove old bundle file %q", entry.Path) - } - - deleted++ - } - } - - // check again - // if the staging dir is not in target path - entries, err = manager.filesystem.List(manager.irodsTempDirPath) - if err != nil { - logger.WithError(err).Warnf("failed to listing a staging directory %q", manager.irodsTempDirPath) - return - } - - if len(entries) == 0 { - // empty - // remove the dir - err := manager.filesystem.RemoveDir(manager.irodsTempDirPath, true, true) - if err != nil { - logger.WithError(err).Warnf("failed to remove a staging directory %q", manager.irodsTempDirPath) - return - } - return - } + err := CleanUpOldIRODSBundles(manager.filesystem, manager.irodsTempDirPath, true, true) + logger.WithError(err).Warnf("failed to clear up staging directory %q", manager.irodsTempDirPath) } func (manager *BundleTransferManager) startProgress() { @@ -559,24 +419,22 @@ func (manager *BundleTransferManager) startProgress() { } func (manager *BundleTransferManager) endProgress() { - if manager.showProgress { - if manager.progressWriter != nil { - manager.mutex.Lock() + if manager.progressWriter != nil { + manager.mutex.Lock() - for _, tracker := range manager.progressTrackers { - if manager.lastError != nil { - tracker.MarkAsDone() - } else { - if !tracker.IsDone() { - tracker.MarkAsErrored() - } + for _, tracker := range manager.progressTrackers { + if manager.lastError != nil { + tracker.MarkAsDone() + } else { + if !tracker.IsDone() { + tracker.MarkAsErrored() } } + } - manager.mutex.Unlock() + manager.mutex.Unlock() - manager.progressWriter.Stop() - } + manager.progressWriter.Stop() } } @@ -655,7 +513,7 @@ func (manager *BundleTransferManager) Start() { } manager.mutex.RUnlock() - if cont && len(bundle.entries) > 0 { + if cont && len(bundle.Entries) > 0 { err := manager.processBundleTar(bundle) if err != nil { // mark error @@ -663,8 +521,8 @@ func (manager *BundleTransferManager) Start() { manager.lastError = err manager.mutex.Unlock() - bundle.lastError = err - bundle.lastErrorTaskName = BundleTaskNameTar + bundle.LastError = err + bundle.LastErrorTaskName = BundleTaskNameTar logger.Error(err) // don't stop here @@ -693,7 +551,7 @@ func (manager *BundleTransferManager) Start() { } manager.mutex.RUnlock() - if cont && len(bundle.entries) > 0 { + if cont && len(bundle.Entries) > 0 { err := manager.processBundleUpload(bundle) if err != nil { // mark error @@ -701,8 +559,8 @@ func (manager *BundleTransferManager) Start() { manager.lastError = err manager.mutex.Unlock() - bundle.lastError = err - bundle.lastErrorTaskName = BundleTaskNameUpload + bundle.LastError = err + bundle.LastErrorTaskName = BundleTaskNameUpload logger.Error(err) // don't stop here @@ -743,7 +601,7 @@ func (manager *BundleTransferManager) Start() { } manager.mutex.RUnlock() - if cont && len(bundle.entries) > 0 { + if cont && len(bundle.Entries) > 0 { err := manager.processBundleRemoveFilesAndMakeDirs(bundle) if err != nil { // mark error @@ -751,8 +609,8 @@ func (manager *BundleTransferManager) Start() { manager.lastError = err manager.mutex.Unlock() - bundle.lastError = err - bundle.lastErrorTaskName = BundleTaskNameRemoveFilesAndMakeDirs + bundle.LastError = err + bundle.LastErrorTaskName = BundleTaskNameRemoveFilesAndMakeDirs logger.Error(err) // don't stop here @@ -779,9 +637,9 @@ func (manager *BundleTransferManager) Start() { case bundle1, ok1 := <-processBundleExtractChan1: if bundle1 != nil { removeTaskCompletedMutex.Lock() - if _, ok := removeTaskCompleted[bundle1.index]; ok { + if _, ok := removeTaskCompleted[bundle1.Index]; ok { // has it - delete(removeTaskCompleted, bundle1.index) + delete(removeTaskCompleted, bundle1.Index) removeTaskCompletedMutex.Unlock() cont := true @@ -792,7 +650,7 @@ func (manager *BundleTransferManager) Start() { } manager.mutex.RUnlock() - if cont && len(bundle1.entries) > 0 { + if cont && len(bundle1.Entries) > 0 { err := manager.processBundleExtract(bundle1) if err != nil { // mark error @@ -800,23 +658,23 @@ func (manager *BundleTransferManager) Start() { manager.lastError = err manager.mutex.Unlock() - bundle1.lastError = err - bundle1.lastErrorTaskName = BundleTaskNameExtract + bundle1.LastError = err + bundle1.LastErrorTaskName = BundleTaskNameExtract logger.Error(err) // don't stop here } } else { - if bundle1.requireTar() { + if bundle1.RequireTar() { // remove irods bundle file - manager.filesystem.RemoveFile(bundle1.irodsBundlePath, true) + manager.filesystem.RemoveFile(bundle1.IRODSBundlePath, true) } } manager.transferWait.Done() } else { - removeTaskCompleted[bundle1.index] = 1 + removeTaskCompleted[bundle1.Index] = 1 removeTaskCompletedMutex.Unlock() } } @@ -828,9 +686,9 @@ func (manager *BundleTransferManager) Start() { case bundle2, ok2 := <-processBundleExtractChan2: if bundle2 != nil { removeTaskCompletedMutex.Lock() - if _, ok := removeTaskCompleted[bundle2.index]; ok { + if _, ok := removeTaskCompleted[bundle2.Index]; ok { // has it - delete(removeTaskCompleted, bundle2.index) + delete(removeTaskCompleted, bundle2.Index) removeTaskCompletedMutex.Unlock() cont := true @@ -841,7 +699,7 @@ func (manager *BundleTransferManager) Start() { } manager.mutex.RUnlock() - if cont && len(bundle2.entries) > 0 { + if cont && len(bundle2.Entries) > 0 { err := manager.processBundleExtract(bundle2) if err != nil { // mark error @@ -849,22 +707,22 @@ func (manager *BundleTransferManager) Start() { manager.lastError = err manager.mutex.Unlock() - bundle2.lastError = err - bundle2.lastErrorTaskName = BundleTaskNameExtract + bundle2.LastError = err + bundle2.LastErrorTaskName = BundleTaskNameExtract logger.Error(err) // don't stop here } } else { - if bundle2.requireTar() { + if bundle2.RequireTar() { // remove irods bundle file - manager.filesystem.RemoveFile(bundle2.irodsBundlePath, true) + manager.filesystem.RemoveFile(bundle2.IRODSBundlePath, true) } } manager.transferWait.Done() } else { - removeTaskCompleted[bundle2.index] = 1 + removeTaskCompleted[bundle2.Index] = 1 removeTaskCompletedMutex.Unlock() } } @@ -901,26 +759,20 @@ func (manager *BundleTransferManager) processBundleRemoveFilesAndMakeDirs(bundle }) // remove files in the bundle if they exist in iRODS - logger.Debugf("deleting exising data objects and creating new collections in the bundle %d", bundle.index) + logger.Debugf("deleting exising data objects and creating new collections in the bundle %d", bundle.Index) progressName := manager.getProgressName(bundle, BundleTaskNameRemoveFilesAndMakeDirs) - totalFileNum := int64(len(bundle.entries)) + totalFileNum := int64(len(bundle.Entries)) processedFiles := int64(0) - if manager.showProgress { - manager.progress(progressName, 0, totalFileNum, progress.UnitsDefault, false) - } + manager.progress(progressName, 0, totalFileNum, progress.UnitsDefault, false) - for _, bundleEntry := range bundle.entries { + for _, bundleEntry := range bundle.Entries { entry, err := manager.filesystem.Stat(bundleEntry.IRODSPath) if err != nil { if !irodsclient_types.IsFileNotFoundError(err) { - if manager.showProgress { - manager.progress(progressName, processedFiles, totalFileNum, progress.UnitsDefault, true) - } - - logger.Error(err) + manager.progress(progressName, processedFiles, totalFileNum, progress.UnitsDefault, true) return xerrors.Errorf("failed to stat data object or collection %q", bundleEntry.IRODSPath) } } @@ -931,11 +783,7 @@ func (manager *BundleTransferManager) processBundleRemoveFilesAndMakeDirs(bundle logger.Debugf("deleting exising collection %q", bundleEntry.IRODSPath) err := manager.filesystem.RemoveDir(bundleEntry.IRODSPath, true, true) if err != nil { - if manager.showProgress { - manager.progress(progressName, processedFiles, totalFileNum, progress.UnitsDefault, true) - } - - logger.Error(err) + manager.progress(progressName, processedFiles, totalFileNum, progress.UnitsDefault, true) return xerrors.Errorf("failed to delete existing collection %q", bundleEntry.IRODSPath) } } @@ -945,23 +793,17 @@ func (manager *BundleTransferManager) processBundleRemoveFilesAndMakeDirs(bundle err := manager.filesystem.RemoveFile(bundleEntry.IRODSPath, true) if err != nil { - if manager.showProgress { - manager.progress(progressName, processedFiles, totalFileNum, progress.UnitsDefault, true) - } - - logger.Error(err) + manager.progress(progressName, processedFiles, totalFileNum, progress.UnitsDefault, true) return xerrors.Errorf("failed to delete existing data object %q", bundleEntry.IRODSPath) } } } processedFiles++ - if manager.showProgress { - manager.progress(progressName, processedFiles, totalFileNum, progress.UnitsDefault, false) - } + manager.progress(progressName, processedFiles, totalFileNum, progress.UnitsDefault, false) } - logger.Debugf("deleted exising data objects in the bundle %d", bundle.index) + logger.Debugf("deleted exising data objects in the bundle %d", bundle.Index) return nil } @@ -972,48 +814,38 @@ func (manager *BundleTransferManager) processBundleTar(bundle *Bundle) error { "function": "processBundleTar", }) - logger.Debugf("creating a tarball for bundle %d to %q", bundle.index, bundle.localBundlePath) + logger.Debugf("creating a tarball for bundle %d to %q", bundle.Index, bundle.LocalBundlePath) progressName := manager.getProgressName(bundle, BundleTaskNameTar) - totalFileNum := int64(len(bundle.entries)) + totalFileNum := int64(len(bundle.Entries)) - var callback func(processed int64, total int64) - if manager.showProgress { - callback = func(processed int64, total int64) { - manager.progress(progressName, processed, total, progress.UnitsDefault, false) - } + callbackTar := func(processed int64, total int64) { + manager.progress(progressName, processed, total, progress.UnitsDefault, false) } - if manager.showProgress { - manager.progress(progressName, 0, totalFileNum, progress.UnitsDefault, false) - } + manager.progress(progressName, 0, totalFileNum, progress.UnitsDefault, false) - if !bundle.requireTar() { + if !bundle.RequireTar() { // no tar, so pass this step - if manager.showProgress { - manager.progress(progressName, totalFileNum, totalFileNum, progress.UnitsDefault, false) - } - - logger.Debugf("skip - creating a tarball for bundle %d to %q", bundle.index, bundle.localBundlePath) + manager.progress(progressName, totalFileNum, totalFileNum, progress.UnitsDefault, false) + logger.Debugf("skip creating a tarball for bundle %d to %q", bundle.Index, bundle.LocalBundlePath) return nil } - entries := make([]string, len(bundle.entries)) - for idx, entry := range bundle.entries { + entries := make([]string, len(bundle.Entries)) + for idx, entry := range bundle.Entries { entries[idx] = entry.LocalPath } - err := Tar(manager.bundleRootPath, entries, bundle.localBundlePath, callback) + err := Tar(manager.bundleRootPath, entries, bundle.LocalBundlePath, callbackTar) if err != nil { - if manager.showProgress { - manager.progress(progressName, 0, totalFileNum, progress.UnitsDefault, true) - } - - return xerrors.Errorf("failed to create a tarball for bundle %d to %q: %w", bundle.index, bundle.localBundlePath, err) + manager.progress(progressName, 0, totalFileNum, progress.UnitsDefault, true) + return xerrors.Errorf("failed to create a tarball for bundle %d to %q: %w", bundle.Index, bundle.LocalBundlePath, err) } - logger.Debugf("created a tarball for bundle %d to %q", bundle.index, bundle.localBundlePath) + manager.progress(progressName, totalFileNum, totalFileNum, progress.UnitsDefault, false) + logger.Debugf("created a tarball for bundle %d to %q", bundle.Index, bundle.LocalBundlePath) return nil } @@ -1024,187 +856,186 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error "function": "processBundleUpload", }) - logger.Debugf("uploading bundle %d to %q", bundle.index, bundle.irodsBundlePath) + logger.Debugf("uploading bundle %d to %q", bundle.Index, bundle.IRODSBundlePath) - progressName := manager.getProgressName(bundle, BundleTaskNameUpload) + if bundle.RequireTar() { + return manager.processBundleUploadWithTar(bundle) + } - totalFileSize := bundle.size + return manager.processBundleUploadWithoutTar(bundle) +} - if bundle.requireTar() { - var callback func(processed int64, total int64) - if manager.showProgress { - callback = func(processed int64, total int64) { - manager.progress(progressName, processed, total, progress.UnitsBytes, false) - } - } +func (manager *BundleTransferManager) processBundleUploadWithTar(bundle *Bundle) error { + logger := log.WithFields(log.Fields{ + "package": "commons", + "struct": "BundleTransferManager", + "function": "processBundleUploadWithTar", + }) - haveExistingBundle := false + progressName := manager.getProgressName(bundle, BundleTaskNameUpload) - bundleEntry, err := manager.filesystem.StatFile(bundle.irodsBundlePath) - if err != nil { - if !irodsclient_types.IsFileNotFoundError(err) { - if manager.showProgress { - manager.progress(progressName, -1, totalFileSize, progress.UnitsBytes, true) - } + callbackPut := func(processed int64, total int64) { + manager.progress(progressName, processed, total, progress.UnitsBytes, false) + } - return xerrors.Errorf("failed to stat existing bundle %q: %w", bundle.irodsBundlePath, err) - } + // check local bundle file + localBundleStat, err := os.Stat(bundle.LocalBundlePath) + if err != nil { + if os.IsNotExist(err) { + manager.progress(progressName, 0, bundle.Size, progress.UnitsBytes, true) + return irodsclient_types.NewFileNotFoundError(bundle.LocalBundlePath) } - if bundleEntry != nil { - localBundleStat, err := os.Stat(bundle.localBundlePath) - if err != nil { - if os.IsNotExist(err) { - return irodsclient_types.NewFileNotFoundError(bundle.localBundlePath) - } - - return xerrors.Errorf("failed to stat %q: %w", bundle.localBundlePath, err) - } + return xerrors.Errorf("failed to stat %q: %w", bundle.LocalBundlePath, err) + } - if bundleEntry.Size == localBundleStat.Size() { - // same file exist - haveExistingBundle = true - } + // check irods bundle file of previous run + bundleEntry, err := manager.filesystem.StatFile(bundle.IRODSBundlePath) + if err != nil { + if !irodsclient_types.IsFileNotFoundError(err) { + manager.progress(progressName, 0, bundle.Size, progress.UnitsBytes, true) + return xerrors.Errorf("failed to stat existing bundle %q: %w", bundle.IRODSBundlePath, err) } + } else { + if bundleEntry.Size == localBundleStat.Size() { + // same file exist + manager.progress(progressName, bundle.Size, bundle.Size, progress.UnitsBytes, false) + // remove local bundle file + os.Remove(bundle.LocalBundlePath) + logger.Debugf("skip uploading bundle %d to %q, file already exists", bundle.Index, bundle.IRODSBundlePath) + return nil + } + } - if !haveExistingBundle { - logger.Debugf("uploading bundle %d to %q", bundle.index, bundle.irodsBundlePath) - - // determine how to download - if manager.singleThreaded || manager.uploadThreadNum == 1 { - _, err = manager.filesystem.UploadFile(bundle.localBundlePath, bundle.irodsBundlePath, "", false, true, true, callback) - } else if manager.redirectToResource { - _, err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, true, true, callback) - } else if manager.useIcat { - _, err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, true, true, callback) + logger.Debugf("uploading bundle %d to %q", bundle.Index, bundle.IRODSBundlePath) + + // determine how to download + if manager.singleThreaded || manager.uploadThreadNum == 1 { + _, err = manager.filesystem.UploadFile(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", false, true, true, callbackPut) + } else if manager.redirectToResource { + _, err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", 0, false, true, true, callbackPut) + } else if manager.useIcat { + _, err = manager.filesystem.UploadFileParallel(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", 0, false, true, true, callbackPut) + } else { + // auto + if bundle.Size >= RedirectToResourceMinSize { + // redirect-to-resource + _, err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", 0, false, true, true, callbackPut) + } else { + if manager.filesystem.SupportParallelUpload() { + _, err = manager.filesystem.UploadFileParallel(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", 0, false, false, false, callbackPut) } else { - // auto - if bundle.size >= RedirectToResourceMinSize { + if bundle.Size >= ParallelUploadMinSize { + // does not support parall upload via iCAT // redirect-to-resource - _, err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, true, true, callback) + _, err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", 0, false, false, false, callbackPut) } else { - if manager.filesystem.SupportParallelUpload() { - _, err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) - } else { - if bundle.size >= ParallelUploadMinSize { - // does not support parall upload via iCAT - // redirect-to-resource - _, err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) - } else { - _, err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, false, false, callback) - } - } + _, err = manager.filesystem.UploadFileParallel(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", 0, false, false, false, callbackPut) } } + } + } - if err != nil { - if manager.showProgress { - manager.progress(progressName, -1, totalFileSize, progress.UnitsBytes, true) - } + if err != nil { + manager.progress(progressName, 0, bundle.Size, progress.UnitsBytes, true) + return xerrors.Errorf("failed to upload bundle %d to %q: %w", bundle.Index, bundle.IRODSBundlePath, err) + } - return xerrors.Errorf("failed to upload bundle %d to %q: %w", bundle.index, bundle.irodsBundlePath, err) - } + // remove local bundle file + os.Remove(bundle.LocalBundlePath) - logger.Debugf("uploaded bundle %d to %q", bundle.index, bundle.irodsBundlePath) - } else { - logger.Debugf("skip uploading bundle %d to %q, file already exists", bundle.index, bundle.irodsBundlePath) - } + logger.Debugf("uploaded bundle %d to %q", bundle.Index, bundle.IRODSBundlePath) - // remove local bundle file - os.Remove(bundle.localBundlePath) - return nil - } + return nil +} - fileUploadProgress := make([]int64, len(bundle.entries)) - fileUploadProgressMutex := sync.Mutex{} +func (manager *BundleTransferManager) processBundleUploadWithoutTar(bundle *Bundle) error { + logger := log.WithFields(log.Fields{ + "package": "commons", + "struct": "BundleTransferManager", + "function": "processBundleUploadWithoutTar", + }) - if manager.showProgress { - manager.progress(progressName, 0, totalFileSize, progress.UnitsBytes, false) - } + progressName := manager.getProgressName(bundle, BundleTaskNameUpload) - for fileIdx, file := range bundle.entries { - var callbackFileUpload func(processed int64, total int64) - if manager.showProgress { - callbackFileUpload = func(processed int64, total int64) { - fileUploadProgressMutex.Lock() - defer fileUploadProgressMutex.Unlock() + fileProgress := make([]int64, len(bundle.Entries)) - fileUploadProgress[fileIdx] = processed + manager.progress(progressName, 0, bundle.Size, progress.UnitsBytes, false) - progressSum := int64(0) - for _, progress := range fileUploadProgress { - progressSum += progress - } + for fileIdx, file := range bundle.Entries { + callbackPut := func(processed int64, total int64) { + fileProgress[fileIdx] = processed - manager.progress(progressName, progressSum, totalFileSize, progress.UnitsBytes, false) + progressSum := int64(0) + for _, progress := range fileProgress { + progressSum += progress } + + manager.progress(progressName, progressSum, bundle.Size, progress.UnitsBytes, false) } - if !manager.filesystem.ExistsDir(path.Dir(file.IRODSPath)) { - // if parent dir does not exist, create - err := manager.filesystem.MakeDir(path.Dir(file.IRODSPath), true) + if file.Dir { + // make dir + err := manager.filesystem.MakeDir(file.IRODSPath, true) if err != nil { - if manager.showProgress { - manager.progress(progressName, -1, totalFileSize, progress.UnitsBytes, true) - } - - return xerrors.Errorf("failed to create a directory %q to upload file %q in bundle %d to %q: %w", path.Dir(file.IRODSPath), file.LocalPath, bundle.index, file.IRODSPath, err) + manager.progress(progressName, 0, bundle.Size, progress.UnitsBytes, true) + return xerrors.Errorf("failed to upload a directory %q in bundle %d to %q: %w", file.LocalPath, bundle.Index, file.IRODSPath, err) } + + manager.progress(progressName, 0, bundle.Size, progress.UnitsBytes, false) + logger.Debugf("uploaded a directory %q in bundle %d to %q", file.LocalPath, bundle.Index, file.IRODSPath) + continue } - var err error - if file.Dir { - err = manager.filesystem.MakeDir(file.IRODSPath, true) + // file + parentDir := path.Dir(file.IRODSPath) + if !manager.filesystem.ExistsDir(parentDir) { + // if parent dir does not exist, create + err := manager.filesystem.MakeDir(parentDir, true) if err != nil { - if manager.showProgress { - manager.progress(progressName, -1, totalFileSize, progress.UnitsBytes, true) - } - - return xerrors.Errorf("failed to upload a directory %q in bundle %d to %q: %w", file.LocalPath, bundle.index, file.IRODSPath, err) + manager.progress(progressName, 0, bundle.Size, progress.UnitsBytes, true) + return xerrors.Errorf("failed to create a directory %q to upload file %q in bundle %d to %q: %w", parentDir, file.LocalPath, bundle.Index, file.IRODSPath, err) } + } - logger.Debugf("uploaded a directory %q in bundle %d to %q", file.LocalPath, bundle.index, file.IRODSPath) + // determine how to download + var err error + if manager.singleThreaded || manager.uploadThreadNum == 1 { + _, err = manager.filesystem.UploadFile(file.LocalPath, file.IRODSPath, "", false, true, true, callbackPut) + } else if manager.redirectToResource { + _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) + } else if manager.useIcat { + _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) } else { - // determine how to download - if manager.singleThreaded || manager.uploadThreadNum == 1 { - _, err = manager.filesystem.UploadFile(file.LocalPath, file.IRODSPath, "", false, false, false, callbackFileUpload) - } else if manager.redirectToResource { - _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) - } else if manager.useIcat { - _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) + // auto + if bundle.Size >= RedirectToResourceMinSize { + // redirect-to-resource + _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) } else { - // auto - if bundle.size >= RedirectToResourceMinSize { - // redirect-to-resource - _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) + if manager.filesystem.SupportParallelUpload() { + _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) } else { - if manager.filesystem.SupportParallelUpload() { - _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) + if bundle.Size >= ParallelUploadMinSize { + // does not support parall upload via iCAT + // redirect-to-resource + _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) } else { - if bundle.size >= ParallelUploadMinSize { - // does not support parall upload via iCAT - // redirect-to-resource - _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) - } else { - _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, false, false, callbackFileUpload) - } + _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) } } } + } - if err != nil { - if manager.showProgress { - manager.progress(progressName, -1, totalFileSize, progress.UnitsBytes, true) - } - - return xerrors.Errorf("failed to upload file %q in bundle %d to %q: %w", file.LocalPath, bundle.index, file.IRODSPath, err) - } - - logger.Debugf("uploaded file %q in bundle %d to %q", file.LocalPath, bundle.index, file.IRODSPath) + if err != nil { + manager.progress(progressName, 0, bundle.Size, progress.UnitsBytes, true) + return xerrors.Errorf("failed to upload file %q in bundle %d to %q: %w", file.LocalPath, bundle.Index, file.IRODSPath, err) } + + manager.progress(progressName, file.Size, bundle.Size, progress.UnitsBytes, false) + logger.Debugf("uploaded file %q in bundle %d to %q", file.LocalPath, bundle.Index, file.IRODSPath) } - logger.Debugf("uploaded files in bundle %d to %q", bundle.index, bundle.irodsBundlePath) + logger.Debugf("uploaded files in bundle %d to %q", bundle.Index, bundle.IRODSBundlePath) return nil } @@ -1215,54 +1046,43 @@ func (manager *BundleTransferManager) processBundleExtract(bundle *Bundle) error "function": "processBundleExtract", }) - logger.Debugf("extracting bundle %d at %q", bundle.index, bundle.irodsBundlePath) + logger.Debugf("extracting bundle %d at %q", bundle.Index, bundle.IRODSBundlePath) progressName := manager.getProgressName(bundle, BundleTaskNameExtract) - totalFileNum := int64(len(bundle.entries)) + totalFileNum := int64(len(bundle.Entries)) - if manager.showProgress { - manager.progress(progressName, 0, totalFileNum, progress.UnitsDefault, false) - } + manager.progress(progressName, 0, totalFileNum, progress.UnitsDefault, false) - if !bundle.requireTar() { + if !bundle.RequireTar() { // no tar, so pass this step - if manager.showProgress { - manager.progress(progressName, totalFileNum, totalFileNum, progress.UnitsDefault, false) - } - - logger.Debugf("skip - extracting bundle %d at %q", bundle.index, bundle.irodsBundlePath) + manager.progress(progressName, totalFileNum, totalFileNum, progress.UnitsDefault, false) + logger.Debugf("skip extracting bundle %d at %q", bundle.Index, bundle.IRODSBundlePath) return nil } - err := manager.filesystem.ExtractStructFile(bundle.irodsBundlePath, manager.irodsDestPath, "", irodsclient_types.TAR_FILE_DT, true, !manager.noBulkRegistration) + err := manager.filesystem.ExtractStructFile(bundle.IRODSBundlePath, manager.irodsDestPath, "", irodsclient_types.TAR_FILE_DT, true, !manager.noBulkRegistration) if err != nil { - if manager.showProgress { - manager.progress(progressName, -1, totalFileNum, progress.UnitsDefault, true) - } - - manager.filesystem.RemoveFile(bundle.irodsBundlePath, true) - return xerrors.Errorf("failed to extract bundle %d at %q to %q: %w", bundle.index, bundle.irodsBundlePath, manager.irodsDestPath, err) + manager.progress(progressName, 0, totalFileNum, progress.UnitsDefault, true) + return xerrors.Errorf("failed to extract bundle %d at %q to %q: %w", bundle.Index, bundle.IRODSBundlePath, manager.irodsDestPath, err) } // remove irods bundle file - logger.Debugf("removing bundle %d at %q", bundle.index, bundle.irodsBundlePath) - manager.filesystem.RemoveFile(bundle.irodsBundlePath, true) + logger.Debugf("removing bundle %d at %q", bundle.Index, bundle.IRODSBundlePath) + manager.filesystem.RemoveFile(bundle.IRODSBundlePath, true) - if manager.showProgress { - manager.progress(progressName, totalFileNum, totalFileNum, progress.UnitsDefault, false) - } + manager.progress(progressName, totalFileNum, totalFileNum, progress.UnitsDefault, false) // set it done - bundle.Done() + bundle.SetCompleted() atomic.AddInt64(&manager.bundlesDoneCounter, 1) - logger.Debugf("extracted bundle %d at %q to %q", bundle.index, bundle.irodsBundlePath, manager.irodsDestPath) + logger.Debugf("extracted bundle %d at %q to %q", bundle.Index, bundle.IRODSBundlePath, manager.irodsDestPath) return nil } func (manager *BundleTransferManager) getProgressName(bundle *Bundle, taskName string) string { - return fmt.Sprintf("bundle %d - %q", bundle.index, taskName) + return fmt.Sprintf("bundle %d - %q", bundle.Index, taskName) } func CleanUpOldLocalBundles(localTempDirPath string, force bool) { From 4322ba5223f5e81c1a86c1feafbe5daef239e238 Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Tue, 20 Aug 2024 15:19:52 -0700 Subject: [PATCH 06/16] rework sync --- cmd/flag/sync.go | 14 ++++- cmd/subcmd/bput.go | 102 +++++++++++++++++++++++++++++++++++-- cmd/subcmd/cp.go | 100 ++++++++++++++++++++++++++++++++++-- cmd/subcmd/get.go | 100 ++++++++++++++++++++++++++++++++++-- cmd/subcmd/put.go | 102 +++++++++++++++++++++++++++++++++++-- cmd/subcmd/sync.go | 31 +++++++---- commons/bundle_transfer.go | 81 +++++++++++++++++++---------- 7 files changed, 477 insertions(+), 53 deletions(-) diff --git a/cmd/flag/sync.go b/cmd/flag/sync.go index f45aef2..e205171 100644 --- a/cmd/flag/sync.go +++ b/cmd/flag/sync.go @@ -5,15 +5,25 @@ import ( ) type SyncFlagValues struct { - Delete bool + Delete bool + BulkUpload bool + Sync bool } var ( syncFlagValues SyncFlagValues ) -func SetSyncFlags(command *cobra.Command) { +func SetSyncFlags(command *cobra.Command, addPutOptionFlag bool) { command.Flags().BoolVar(&syncFlagValues.Delete, "delete", false, "Delete extra files in dest dir") + + if addPutOptionFlag { + command.Flags().BoolVar(&syncFlagValues.BulkUpload, "bulk_upload", false, "Use bulk upload") + } + + // this is hidden + command.Flags().BoolVar(&syncFlagValues.Sync, "sync", false, "Set this for sync") + command.Flags().MarkHidden("sync") } func GetSyncFlagValues() *SyncFlagValues { diff --git a/cmd/subcmd/bput.go b/cmd/subcmd/bput.go index 5da939f..03d415a 100644 --- a/cmd/subcmd/bput.go +++ b/cmd/subcmd/bput.go @@ -40,7 +40,7 @@ func AddBputCommand(rootCmd *cobra.Command) { flag.SetRetryFlags(bputCmd) flag.SetDifferentialTransferFlags(bputCmd, true) flag.SetNoRootFlags(bputCmd) - flag.SetSyncFlags(bputCmd) + flag.SetSyncFlags(bputCmd, false) flag.SetTransferReportFlags(putCmd) rootCmd.AddCommand(bputCmd) @@ -207,7 +207,7 @@ func (bput *BputCommand) Process() error { } // bundle transfer manager - bput.bundleTransferManager = commons.NewBundleTransferManager(bput.filesystem, bput.targetPath, bundleRootPath, bput.bundleTransferFlagValues.MinFileNum, bput.bundleTransferFlagValues.MaxFileNum, bput.bundleTransferFlagValues.MaxFileSize, bput.parallelTransferFlagValues.SingleTread, bput.parallelTransferFlagValues.ThreadNumber, bput.parallelTransferFlagValues.RedirectToResource, bput.parallelTransferFlagValues.Icat, bput.bundleTransferFlagValues.LocalTempPath, bput.bundleTransferFlagValues.IRODSTempPath, bput.bundleTransferFlagValues.NoBulkRegistration, bput.progressFlagValues.ShowProgress, bput.progressFlagValues.ShowFullPath) + bput.bundleTransferManager = commons.NewBundleTransferManager(bput.filesystem, bput.transferReportManager, bput.targetPath, bundleRootPath, bput.bundleTransferFlagValues.MinFileNum, bput.bundleTransferFlagValues.MaxFileNum, bput.bundleTransferFlagValues.MaxFileSize, bput.parallelTransferFlagValues.SingleTread, bput.parallelTransferFlagValues.ThreadNumber, bput.parallelTransferFlagValues.RedirectToResource, bput.parallelTransferFlagValues.Icat, bput.bundleTransferFlagValues.LocalTempPath, bput.bundleTransferFlagValues.IRODSTempPath, bput.bundleTransferFlagValues.NoBulkRegistration, bput.progressFlagValues.ShowProgress, bput.progressFlagValues.ShowFullPath) bput.bundleTransferManager.Start() // run @@ -431,7 +431,54 @@ func (bput *BputCommand) putFile(sourceStat fs.FileInfo, sourcePath string) erro // target exists // target must be a file if targetEntry.IsDir() { - return commons.NewNotFileError(targetPath) + if bput.syncFlagValues.Sync { + // if it is sync, remove + if bput.forceFlagValues.Force { + removeErr := bput.filesystem.RemoveDir(targetPath, true, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "put", "dir"}, + } + + bput.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + // ask + overwrite := commons.InputYN(fmt.Sprintf("overwriting a file %q, but directory exists. Overwrite?", targetPath)) + if overwrite { + removeErr := bput.filesystem.RemoveDir(targetPath, true, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "put", "dir"}, + } + + bput.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + return commons.NewNotFileError(targetPath) + } + } + } else { + return commons.NewNotFileError(targetPath) + } } if bput.differentialTransferFlagValues.DifferentialTransfer { @@ -560,7 +607,54 @@ func (bput *BputCommand) putDir(sourceStat fs.FileInfo, sourcePath string) error } else { // target exists if !targetEntry.IsDir() { - return commons.NewNotDirError(targetPath) + if bput.syncFlagValues.Sync { + // if it is sync, remove + if bput.forceFlagValues.Force { + removeErr := bput.filesystem.RemoveFile(targetPath, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "put"}, + } + + bput.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + // ask + overwrite := commons.InputYN(fmt.Sprintf("overwriting a directory %q, but file exists. Overwrite?", targetPath)) + if overwrite { + removeErr := bput.filesystem.RemoveFile(targetPath, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "put"}, + } + + bput.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + return commons.NewNotDirError(targetPath) + } + } + } else { + return commons.NewNotDirError(targetPath) + } } } diff --git a/cmd/subcmd/cp.go b/cmd/subcmd/cp.go index 1def183..e957cea 100644 --- a/cmd/subcmd/cp.go +++ b/cmd/subcmd/cp.go @@ -36,7 +36,7 @@ func AddCpCommand(rootCmd *cobra.Command) { flag.SetRetryFlags(cpCmd) flag.SetDifferentialTransferFlags(cpCmd, true) flag.SetNoRootFlags(cpCmd) - flag.SetSyncFlags(cpCmd) + flag.SetSyncFlags(cpCmd, false) flag.SetTransferReportFlags(getCmd) rootCmd.AddCommand(cpCmd) @@ -315,7 +315,54 @@ func (cp *CpCommand) copyFile(sourceEntry *irodsclient_fs.Entry, targetPath stri // target exists // target must be a file if targetEntry.IsDir() { - return commons.NewNotFileError(targetPath) + if cp.syncFlagValues.Sync { + // if it is sync, remove + if cp.forceFlagValues.Force { + removeErr := cp.filesystem.RemoveDir(targetPath, true, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "cp", "dir"}, + } + + cp.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + // ask + overwrite := commons.InputYN(fmt.Sprintf("overwriting a file %q, but directory exists. Overwrite?", targetPath)) + if overwrite { + removeErr := cp.filesystem.RemoveDir(targetPath, true, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "cp", "dir"}, + } + + cp.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + return commons.NewNotFileError(targetPath) + } + } + } else { + return commons.NewNotFileError(targetPath) + } } if cp.differentialTransferFlagValues.DifferentialTransfer { @@ -433,7 +480,54 @@ func (cp *CpCommand) copyDir(sourceEntry *irodsclient_fs.Entry, targetPath strin } else { // target exists if !targetEntry.IsDir() { - return commons.NewNotDirError(targetPath) + if cp.syncFlagValues.Sync { + // if it is sync, remove + if cp.forceFlagValues.Force { + removeErr := cp.filesystem.RemoveFile(targetPath, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "cp"}, + } + + cp.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + // ask + overwrite := commons.InputYN(fmt.Sprintf("overwriting a directory %q, but file exists. Overwrite?", targetPath)) + if overwrite { + removeErr := cp.filesystem.RemoveFile(targetPath, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "cp"}, + } + + cp.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + return commons.NewNotDirError(targetPath) + } + } + } else { + return commons.NewNotDirError(targetPath) + } } } diff --git a/cmd/subcmd/get.go b/cmd/subcmd/get.go index 62bc7ee..82e1e99 100644 --- a/cmd/subcmd/get.go +++ b/cmd/subcmd/get.go @@ -42,7 +42,7 @@ func AddGetCommand(rootCmd *cobra.Command) { flag.SetChecksumFlags(getCmd, false) flag.SetTransferReportFlags(getCmd) flag.SetNoRootFlags(getCmd) - flag.SetSyncFlags(getCmd) + flag.SetSyncFlags(getCmd, false) flag.SetDecryptionFlags(getCmd) flag.SetPostTransferFlagValues(getCmd) @@ -454,7 +454,54 @@ func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath strin // target exists // target must be a file if targetStat.IsDir() { - return commons.NewNotFileError(targetPath) + if get.syncFlagValues.Sync { + // if it is sync, remove + if get.forceFlagValues.Force { + removeErr := os.RemoveAll(targetPath) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "get", "dir"}, + } + + get.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + // ask + overwrite := commons.InputYN(fmt.Sprintf("overwriting a file %q, but directory exists. Overwrite?", targetPath)) + if overwrite { + removeErr := os.RemoveAll(targetPath) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "get", "dir"}, + } + + get.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + return commons.NewNotFileError(targetPath) + } + } + } else { + return commons.NewNotFileError(targetPath) + } } // check transfer status file @@ -589,7 +636,54 @@ func (get *GetCommand) getDir(sourceEntry *irodsclient_fs.Entry, targetPath stri } else { // target exists if !targetStat.IsDir() { - return commons.NewNotDirError(targetPath) + if get.syncFlagValues.Sync { + // if it is sync, remove + if get.forceFlagValues.Force { + removeErr := os.Remove(targetPath) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "get"}, + } + + get.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + // ask + overwrite := commons.InputYN(fmt.Sprintf("overwriting a directory %q, but file exists. Overwrite?", targetPath)) + if overwrite { + removeErr := os.Remove(targetPath) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "put"}, + } + + get.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + return commons.NewNotDirError(targetPath) + } + } + } else { + return commons.NewNotDirError(targetPath) + } } } diff --git a/cmd/subcmd/put.go b/cmd/subcmd/put.go index a0a6ec6..e71f5fa 100644 --- a/cmd/subcmd/put.go +++ b/cmd/subcmd/put.go @@ -42,7 +42,7 @@ func AddPutCommand(rootCmd *cobra.Command) { flag.SetDifferentialTransferFlags(putCmd, true) flag.SetChecksumFlags(putCmd, true) flag.SetNoRootFlags(putCmd) - flag.SetSyncFlags(putCmd) + flag.SetSyncFlags(putCmd, false) flag.SetEncryptionFlags(putCmd) flag.SetPostTransferFlagValues(putCmd) flag.SetTransferReportFlags(putCmd) @@ -464,7 +464,54 @@ func (put *PutCommand) putFile(sourceStat fs.FileInfo, sourcePath string, tempPa // target exists // target must be a file if targetEntry.IsDir() { - return commons.NewNotFileError(targetPath) + if put.syncFlagValues.Sync { + // if it is sync, remove + if put.forceFlagValues.Force { + removeErr := put.filesystem.RemoveDir(targetPath, true, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "put", "dir"}, + } + + put.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + // ask + overwrite := commons.InputYN(fmt.Sprintf("overwriting a file %q, but directory exists. Overwrite?", targetPath)) + if overwrite { + removeErr := put.filesystem.RemoveDir(targetPath, true, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "put", "dir"}, + } + + put.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + return commons.NewNotFileError(targetPath) + } + } + } else { + return commons.NewNotFileError(targetPath) + } } if put.differentialTransferFlagValues.DifferentialTransfer { @@ -566,7 +613,7 @@ func (put *PutCommand) putDir(sourceStat fs.FileInfo, sourcePath string, targetP if err != nil { if irodsclient_types.IsFileNotFoundError(err) { // target does not exist - // target must be a directorywith new name + // target must be a directory with new name err = put.filesystem.MakeDir(targetPath, true) if err != nil { return xerrors.Errorf("failed to make a collection %q: %w", targetPath, err) @@ -589,7 +636,54 @@ func (put *PutCommand) putDir(sourceStat fs.FileInfo, sourcePath string, targetP } else { // target exists if !targetEntry.IsDir() { - return commons.NewNotDirError(targetPath) + if put.syncFlagValues.Sync { + // if it is sync, remove + if put.forceFlagValues.Force { + removeErr := put.filesystem.RemoveFile(targetPath, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "put"}, + } + + put.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + // ask + overwrite := commons.InputYN(fmt.Sprintf("overwriting a directory %q, but file exists. Overwrite?", targetPath)) + if overwrite { + removeErr := put.filesystem.RemoveFile(targetPath, true) + + now := time.Now() + reportFile := &commons.TransferReportFile{ + Method: commons.TransferMethodDelete, + StartAt: now, + EndAt: now, + SourcePath: targetPath, + Error: removeErr, + Notes: []string{"overwrite", "put"}, + } + + put.transferReportManager.AddFile(reportFile) + + if removeErr != nil { + return removeErr + } + } else { + return commons.NewNotDirError(targetPath) + } + } + } else { + return commons.NewNotDirError(targetPath) + } } } diff --git a/cmd/subcmd/sync.go b/cmd/subcmd/sync.go index f7a1751..50579cd 100644 --- a/cmd/subcmd/sync.go +++ b/cmd/subcmd/sync.go @@ -31,7 +31,7 @@ func AddSyncCommand(rootCmd *cobra.Command) { flag.SetRetryFlags(syncCmd) flag.SetDifferentialTransferFlags(syncCmd, false) flag.SetNoRootFlags(syncCmd) - flag.SetSyncFlags(syncCmd) + flag.SetSyncFlags(syncCmd, true) rootCmd.AddCommand(syncCmd) } @@ -49,6 +49,7 @@ type SyncCommand struct { command *cobra.Command retryFlagValues *flag.RetryFlagValues + syncFlagValues *flag.SyncFlagValues sourcePaths []string targetPath string @@ -59,6 +60,7 @@ func NewSyncCommand(command *cobra.Command, args []string) (*SyncCommand, error) command: command, retryFlagValues: flag.GetRetryFlagValues(), + syncFlagValues: flag.GetSyncFlagValues(), } // path @@ -159,7 +161,7 @@ func (sync *SyncCommand) syncIRODS(targetPath string) error { return nil } -func (sync *SyncCommand) getNewCommandArgsForRetry() ([]string, error) { +func (sync *SyncCommand) getNewCommandArgs() ([]string, error) { newArgs := []string{} commandName := sync.command.CalledAs() @@ -179,6 +181,7 @@ func (sync *SyncCommand) getNewCommandArgsForRetry() ([]string, error) { newArgs = append(newArgs, osArgs[:commandIdx]...) newArgs = append(newArgs, "--diff") + newArgs = append(newArgs, "--sync") newArgs = append(newArgs, osArgs[commandIdx+1:]...) // filter out retry flag @@ -199,16 +202,24 @@ func (sync *SyncCommand) syncLocalToIRODS() error { "function": "syncLocalToIRODS", }) - newArgs, err := sync.getNewCommandArgsForRetry() + newArgs, err := sync.getNewCommandArgs() if err != nil { return xerrors.Errorf("failed to get new command args for retry: %w", err) } - // run bput - logger.Debugf("run bput with args: %v", newArgs) - bputCmd.ParseFlags(newArgs) - argWoFlags := bputCmd.Flags().Args() - return bputCmd.RunE(bputCmd, argWoFlags) + if sync.syncFlagValues.BulkUpload { + // run bput + logger.Debugf("run bput with args: %v", newArgs) + bputCmd.ParseFlags(newArgs) + argWoFlags := bputCmd.Flags().Args() + return bputCmd.RunE(bputCmd, argWoFlags) + } + + // run put + logger.Debugf("run put with args: %v", newArgs) + putCmd.ParseFlags(newArgs) + argWoFlags := putCmd.Flags().Args() + return putCmd.RunE(putCmd, argWoFlags) } func (sync *SyncCommand) syncIRODSToIRODS() error { @@ -218,7 +229,7 @@ func (sync *SyncCommand) syncIRODSToIRODS() error { "function": "syncIRODSToIRODS", }) - newArgs, err := sync.getNewCommandArgsForRetry() + newArgs, err := sync.getNewCommandArgs() if err != nil { return xerrors.Errorf("failed to get new command args for retry: %w", err) } @@ -237,7 +248,7 @@ func (sync *SyncCommand) syncIRODSToLocal() error { "function": "syncIRODSToLocal", }) - newArgs, err := sync.getNewCommandArgsForRetry() + newArgs, err := sync.getNewCommandArgs() if err != nil { return xerrors.Errorf("failed to get new command args for retry: %w", err) } diff --git a/commons/bundle_transfer.go b/commons/bundle_transfer.go index 03bb3a7..0ed296d 100644 --- a/commons/bundle_transfer.go +++ b/commons/bundle_transfer.go @@ -9,6 +9,7 @@ import ( "path/filepath" "sync" "sync/atomic" + "time" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" @@ -153,6 +154,7 @@ func (bundle *Bundle) SetCompleted() { type BundleTransferManager struct { filesystem *irodsclient_fs.FileSystem + transferReportManager *TransferReportManager irodsDestPath string currentBundle *Bundle nextBundleIndex int64 @@ -185,7 +187,7 @@ type BundleTransferManager struct { } // NewBundleTransferManager creates a new BundleTransferManager -func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, irodsDestPath string, bundleRootPath string, minBundleFileNum int, maxBundleFileNum int, maxBundleFileSize int64, singleThreaded bool, uploadThreadNum int, redirectToResource bool, useIcat bool, localTempDirPath string, irodsTempDirPath string, noBulkReg bool, showProgress bool, showFullPath bool) *BundleTransferManager { +func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, transferReportManager *TransferReportManager, irodsDestPath string, bundleRootPath string, minBundleFileNum int, maxBundleFileNum int, maxBundleFileSize int64, singleThreaded bool, uploadThreadNum int, redirectToResource bool, useIcat bool, localTempDirPath string, irodsTempDirPath string, noBulkReg bool, showProgress bool, showFullPath bool) *BundleTransferManager { cwd := GetCWD() home := GetHomeDir() zone := GetZone() @@ -193,6 +195,7 @@ func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, irodsDestPath strin manager := &BundleTransferManager{ filesystem: fs, + transferReportManager: transferReportManager, irodsDestPath: irodsDestPath, currentBundle: nil, nextBundleIndex: 0, @@ -922,17 +925,7 @@ func (manager *BundleTransferManager) processBundleUploadWithTar(bundle *Bundle) // redirect-to-resource _, err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", 0, false, true, true, callbackPut) } else { - if manager.filesystem.SupportParallelUpload() { - _, err = manager.filesystem.UploadFileParallel(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", 0, false, false, false, callbackPut) - } else { - if bundle.Size >= ParallelUploadMinSize { - // does not support parall upload via iCAT - // redirect-to-resource - _, err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", 0, false, false, false, callbackPut) - } else { - _, err = manager.filesystem.UploadFileParallel(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", 0, false, false, false, callbackPut) - } - } + _, err = manager.filesystem.UploadFileParallel(bundle.LocalBundlePath, bundle.IRODSBundlePath, "", 0, false, false, false, callbackPut) } } @@ -982,6 +975,18 @@ func (manager *BundleTransferManager) processBundleUploadWithoutTar(bundle *Bund return xerrors.Errorf("failed to upload a directory %q in bundle %d to %q: %w", file.LocalPath, bundle.Index, file.IRODSPath, err) } + now := time.Now() + reportFile := &TransferReportFile{ + Method: TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: file.LocalPath, + DestPath: file.IRODSPath, + Notes: []string{"directory"}, + } + + manager.transferReportManager.AddFile(reportFile) + manager.progress(progressName, 0, bundle.Size, progress.UnitsBytes, false) logger.Debugf("uploaded a directory %q in bundle %d to %q", file.LocalPath, bundle.Index, file.IRODSPath) continue @@ -998,31 +1003,29 @@ func (manager *BundleTransferManager) processBundleUploadWithoutTar(bundle *Bund } } + var uploadResult *irodsclient_fs.FileTransferResult + notes := []string{} + // determine how to download var err error if manager.singleThreaded || manager.uploadThreadNum == 1 { - _, err = manager.filesystem.UploadFile(file.LocalPath, file.IRODSPath, "", false, true, true, callbackPut) + uploadResult, err = manager.filesystem.UploadFile(file.LocalPath, file.IRODSPath, "", false, true, true, callbackPut) + notes = append(notes, "icat", "single-thread") } else if manager.redirectToResource { - _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) + uploadResult, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) + notes = append(notes, "redirect-to-resource") } else if manager.useIcat { - _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) + uploadResult, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) + notes = append(notes, "icat", "multi-thread") } else { // auto if bundle.Size >= RedirectToResourceMinSize { // redirect-to-resource - _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) + uploadResult, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) + notes = append(notes, "redirect-to-resource") } else { - if manager.filesystem.SupportParallelUpload() { - _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) - } else { - if bundle.Size >= ParallelUploadMinSize { - // does not support parall upload via iCAT - // redirect-to-resource - _, err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) - } else { - _, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) - } - } + uploadResult, err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, true, true, callbackPut) + notes = append(notes, "icat", "multi-thread") } } @@ -1031,6 +1034,12 @@ func (manager *BundleTransferManager) processBundleUploadWithoutTar(bundle *Bund return xerrors.Errorf("failed to upload file %q in bundle %d to %q: %w", file.LocalPath, bundle.Index, file.IRODSPath, err) } + err = manager.transferReportManager.AddTransfer(uploadResult, TransferMethodPut, err, notes) + if err != nil { + manager.progress(progressName, 0, bundle.Size, progress.UnitsBytes, true) + return xerrors.Errorf("failed to add transfer report: %w", err) + } + manager.progress(progressName, file.Size, bundle.Size, progress.UnitsBytes, false) logger.Debugf("uploaded file %q in bundle %d to %q", file.LocalPath, bundle.Index, file.IRODSPath) } @@ -1077,6 +1086,24 @@ func (manager *BundleTransferManager) processBundleExtract(bundle *Bundle) error bundle.SetCompleted() atomic.AddInt64(&manager.bundlesDoneCounter, 1) + now := time.Now() + + for _, file := range bundle.Entries { + reportFile := &TransferReportFile{ + Method: TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: file.LocalPath, + SourceSize: file.Size, + + DestPath: file.IRODSPath, + DestSize: file.Size, + Notes: []string{"bundle_extracted"}, + } + + manager.transferReportManager.AddFile(reportFile) + } + logger.Debugf("extracted bundle %d at %q to %q", bundle.Index, bundle.IRODSBundlePath, manager.irodsDestPath) return nil } From 50995d0cca147bba5723f9f60d21350c05c156df Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Tue, 20 Aug 2024 16:01:19 -0700 Subject: [PATCH 07/16] fix typo --- cmd/subcmd/bput.go | 2 +- cmd/subcmd/cp.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/subcmd/bput.go b/cmd/subcmd/bput.go index 03d415a..a14806b 100644 --- a/cmd/subcmd/bput.go +++ b/cmd/subcmd/bput.go @@ -41,7 +41,7 @@ func AddBputCommand(rootCmd *cobra.Command) { flag.SetDifferentialTransferFlags(bputCmd, true) flag.SetNoRootFlags(bputCmd) flag.SetSyncFlags(bputCmd, false) - flag.SetTransferReportFlags(putCmd) + flag.SetTransferReportFlags(bputCmd) rootCmd.AddCommand(bputCmd) } diff --git a/cmd/subcmd/cp.go b/cmd/subcmd/cp.go index e957cea..6d5fe09 100644 --- a/cmd/subcmd/cp.go +++ b/cmd/subcmd/cp.go @@ -37,7 +37,7 @@ func AddCpCommand(rootCmd *cobra.Command) { flag.SetDifferentialTransferFlags(cpCmd, true) flag.SetNoRootFlags(cpCmd) flag.SetSyncFlags(cpCmd, false) - flag.SetTransferReportFlags(getCmd) + flag.SetTransferReportFlags(cpCmd) rootCmd.AddCommand(cpCmd) } From 0315ca07ec3bbf5816d1aab61f9548ef0978d973 Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Tue, 20 Aug 2024 16:34:05 -0700 Subject: [PATCH 08/16] fix various bugs --- cmd/flag/encryption.go | 4 +-- cmd/subcmd/bput.go | 5 +++ cmd/subcmd/cp.go | 5 +++ cmd/subcmd/get.go | 65 ++++++++++++++++---------------------- cmd/subcmd/ls.go | 45 ++++++++++---------------- cmd/subcmd/mv.go | 5 +++ cmd/subcmd/put.go | 5 +++ commons/bundle_transfer.go | 2 +- commons/parallel.go | 5 +-- 9 files changed, 69 insertions(+), 72 deletions(-) diff --git a/cmd/flag/encryption.go b/cmd/flag/encryption.go index dc58feb..f2b9d4f 100644 --- a/cmd/flag/encryption.go +++ b/cmd/flag/encryption.go @@ -21,7 +21,6 @@ type EncryptionFlagValues struct { type DecryptionFlagValues struct { Decryption bool NoDecryption bool - IgnoreMeta bool Key string PrivateKeyPath string TempPath string @@ -43,9 +42,8 @@ func SetEncryptionFlags(command *cobra.Command) { } func SetDecryptionFlags(command *cobra.Command) { - command.Flags().BoolVar(&decryptionFlagValues.Decryption, "decrypt", false, "Decrypt files") + command.Flags().BoolVar(&decryptionFlagValues.Decryption, "decrypt", true, "Decrypt files") command.Flags().BoolVar(&decryptionFlagValues.NoDecryption, "no_decrypt", false, "Disable decryption forcefully") - command.Flags().BoolVar(&decryptionFlagValues.IgnoreMeta, "ignore_meta", false, "Ignore decryption config via metadata") command.Flags().StringVar(&decryptionFlagValues.Key, "decrypt_key", "", "Decryption key for 'winscp' and 'pgp' mode") command.Flags().StringVar(&decryptionFlagValues.PrivateKeyPath, "decrypt_priv_key", commons.GetDefaultPrivateKeyPath(), "Decryption private key for 'ssh' mode") command.Flags().StringVar(&decryptionFlagValues.TempPath, "decrypt_temp", os.TempDir(), "Specify temp directory path for decrypting files") diff --git a/cmd/subcmd/bput.go b/cmd/subcmd/bput.go index a14806b..a0297e8 100644 --- a/cmd/subcmd/bput.go +++ b/cmd/subcmd/bput.go @@ -257,6 +257,11 @@ func (bput *BputCommand) ensureTargetIsDir(targetPath string) error { targetEntry, err := bput.filesystem.Stat(targetPath) if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // not exist + return commons.NewNotDirError(targetPath) + } + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } diff --git a/cmd/subcmd/cp.go b/cmd/subcmd/cp.go index 6d5fe09..4500ee9 100644 --- a/cmd/subcmd/cp.go +++ b/cmd/subcmd/cp.go @@ -196,6 +196,11 @@ func (cp *CpCommand) ensureTargetIsDir(targetPath string) error { targetEntry, err := cp.filesystem.Stat(targetPath) if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // not exist + return commons.NewNotDirError(targetPath) + } + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } diff --git a/cmd/subcmd/get.go b/cmd/subcmd/get.go index 82e1e99..6bedb7e 100644 --- a/cmd/subcmd/get.go +++ b/cmd/subcmd/get.go @@ -248,6 +248,11 @@ func (get *GetCommand) ensureTargetIsDir(targetPath string) error { targetStat, err := os.Stat(targetPath) if err != nil { + if os.IsNotExist(err) { + // not exist + return commons.NewNotDirError(targetPath) + } + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } @@ -258,39 +263,24 @@ func (get *GetCommand) ensureTargetIsDir(targetPath string) error { return nil } -func (get *GetCommand) requireDecryption(sourceEntry *irodsclient_fs.Entry, parentDecryption bool) bool { - if get.decryptionFlagValues.Decryption { - return true - } - +func (get *GetCommand) requireDecryption(sourcePath string) bool { if get.decryptionFlagValues.NoDecryption { return false } - if !get.decryptionFlagValues.IgnoreMeta { - // load encryption config from meta - sourceDir := sourceEntry.Path - if !sourceEntry.IsDir() { - sourceDir = commons.GetDir(sourceEntry.Path) - } - - encryptionConfig := commons.GetEncryptionConfigFromMeta(get.filesystem, sourceDir) - - return encryptionConfig.Required + if !get.decryptionFlagValues.Decryption { + return false } - return parentDecryption + mode := commons.DetectEncryptionMode(sourcePath) + return mode != commons.EncryptionModeUnknown } func (get *GetCommand) hasTransferStatusFile(targetPath string) bool { // check transfer status file trxStatusFilePath := irodsclient_irodsfs.GetDataObjectTransferStatusFilePath(targetPath) _, err := os.Stat(trxStatusFilePath) - if err == nil { - return true - } - - return false + return err == nil } func (get *GetCommand) getOne(sourcePath string, targetPath string) error { @@ -311,26 +301,25 @@ func (get *GetCommand) getOne(sourcePath string, targetPath string) error { targetPath = commons.MakeTargetLocalFilePath(sourcePath, targetPath) } - return get.getDir(sourceEntry, targetPath, false) + return get.getDir(sourceEntry, targetPath) } // file - requireDecryption := get.requireDecryption(sourceEntry, false) - if requireDecryption { + if get.requireDecryption(sourceEntry.Path) { // decrypt filename tempPath, newTargetPath, err := get.getPathsForDecryption(sourceEntry.Path, targetPath) if err != nil { return xerrors.Errorf("failed to get decryption path for %q: %w", sourceEntry.Path, err) } - return get.getFile(sourceEntry, tempPath, newTargetPath, requireDecryption) + return get.getFile(sourceEntry, tempPath, newTargetPath) } targetPath = commons.MakeTargetLocalFilePath(sourcePath, targetPath) - return get.getFile(sourceEntry, "", targetPath, requireDecryption) + return get.getFile(sourceEntry, "", targetPath) } -func (get *GetCommand) scheduleGet(sourceEntry *irodsclient_fs.Entry, tempPath string, targetPath string, resume bool, requireDecryption bool) error { +func (get *GetCommand) scheduleGet(sourceEntry *irodsclient_fs.Entry, tempPath string, targetPath string, resume bool) error { logger := log.WithFields(log.Fields{ "package": "subcmd", "struct": "GetCommand", @@ -396,7 +385,7 @@ func (get *GetCommand) scheduleGet(sourceEntry *irodsclient_fs.Entry, tempPath s } // decrypt - if requireDecryption { + if get.requireDecryption(sourceEntry.Path) { decrypted, err := get.decryptFile(sourceEntry.Path, tempPath, targetPath) if err != nil { job.Progress(-1, sourceEntry.Size, true) @@ -417,6 +406,8 @@ func (get *GetCommand) scheduleGet(sourceEntry *irodsclient_fs.Entry, tempPath s logger.Debugf("downloaded a data object %q to %q", sourceEntry.Path, targetPath) job.Progress(sourceEntry.Size, sourceEntry.Size, false) + job.Done() + return nil } @@ -431,7 +422,7 @@ func (get *GetCommand) scheduleGet(sourceEntry *irodsclient_fs.Entry, tempPath s return nil } -func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath string, targetPath string, requireDecryption bool) error { +func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath string, targetPath string) error { logger := log.WithFields(log.Fields{ "package": "subcmd", "struct": "GetCommand", @@ -445,7 +436,7 @@ func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath strin if os.IsNotExist(err) { // target does not exist // target must be a file with new name - return get.scheduleGet(sourceEntry, tempPath, targetPath, false, requireDecryption) + return get.scheduleGet(sourceEntry, tempPath, targetPath, false) } return xerrors.Errorf("failed to stat %q: %w", targetPath, err) @@ -510,7 +501,7 @@ func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath strin commons.Printf("resume downloading a data object %q\n", targetPath) logger.Debugf("resume downloading a data object %q", targetPath) - return get.scheduleGet(sourceEntry, tempPath, targetPath, true, requireDecryption) + return get.scheduleGet(sourceEntry, tempPath, targetPath, true) } if get.differentialTransferFlagValues.DifferentialTransfer { @@ -603,10 +594,10 @@ func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath strin } // schedule - return get.scheduleGet(sourceEntry, tempPath, targetPath, false, requireDecryption) + return get.scheduleGet(sourceEntry, tempPath, targetPath, false) } -func (get *GetCommand) getDir(sourceEntry *irodsclient_fs.Entry, targetPath string, parentDecryption bool) error { +func (get *GetCommand) getDir(sourceEntry *irodsclient_fs.Entry, targetPath string) error { commons.MarkPathMap(get.updatedPathMap, targetPath) targetStat, err := os.Stat(targetPath) @@ -688,7 +679,7 @@ func (get *GetCommand) getDir(sourceEntry *irodsclient_fs.Entry, targetPath stri } // load encryption config - requireDecryption := get.requireDecryption(sourceEntry, parentDecryption) + requireDecryption := get.requireDecryption(sourceEntry.Path) // get entries entries, err := get.filesystem.List(sourceEntry.Path) @@ -701,7 +692,7 @@ func (get *GetCommand) getDir(sourceEntry *irodsclient_fs.Entry, targetPath stri if entry.IsDir() { // dir - err = get.getDir(entry, newEntryPath, requireDecryption) + err = get.getDir(entry, newEntryPath) if err != nil { return err } @@ -714,12 +705,12 @@ func (get *GetCommand) getDir(sourceEntry *irodsclient_fs.Entry, targetPath stri return xerrors.Errorf("failed to get decryption path for %q: %w", entry.Path, err) } - err = get.getFile(entry, tempPath, newTargetPath, requireDecryption) + err = get.getFile(entry, tempPath, newTargetPath) if err != nil { return err } } else { - err = get.getFile(entry, "", newEntryPath, requireDecryption) + err = get.getFile(entry, "", newEntryPath) if err != nil { return err } diff --git a/cmd/subcmd/ls.go b/cmd/subcmd/ls.go index 7420313..87f8fa6 100644 --- a/cmd/subcmd/ls.go +++ b/cmd/subcmd/ls.go @@ -147,28 +147,17 @@ func (ls *LsCommand) Process() error { return nil } -func (ls *LsCommand) requireDecryption(sourceEntry *irodsclient_fs.Entry, parentDecryption bool) bool { - if ls.decryptionFlagValues.Decryption { - return true - } - +func (ls *LsCommand) requireDecryption(sourcePath string) bool { if ls.decryptionFlagValues.NoDecryption { return false } - if !ls.decryptionFlagValues.IgnoreMeta { - // load encryption config from meta - sourceDir := sourceEntry.Path - if !sourceEntry.IsDir() { - sourceDir = commons.GetDir(sourceEntry.Path) - } - - encryptionConfig := commons.GetEncryptionConfigFromMeta(ls.filesystem, sourceDir) - - return encryptionConfig.Required + if !ls.decryptionFlagValues.Decryption { + return false } - return parentDecryption + mode := commons.DetectEncryptionMode(sourcePath) + return mode != commons.EncryptionModeUnknown } func (ls *LsCommand) listOne(sourcePath string) error { @@ -186,8 +175,6 @@ func (ls *LsCommand) listOne(sourcePath string) error { return xerrors.Errorf("failed to stat %q: %w", sourcePath, err) } - requireDecryption := ls.requireDecryption(sourceEntry, false) - connection, err := ls.filesystem.GetMetadataConnection() if err != nil { return xerrors.Errorf("failed to get connection: %w", err) @@ -211,7 +198,7 @@ func (ls *LsCommand) listOne(sourcePath string) error { return xerrors.Errorf("failed to list data-objects in %q: %w", sourcePath, err) } - ls.printDataObjects(objs, requireDecryption) + ls.printDataObjects(objs) ls.printCollections(colls) return nil @@ -231,7 +218,7 @@ func (ls *LsCommand) listOne(sourcePath string) error { } entries := []*irodsclient_types.IRODSDataObject{entry} - ls.printDataObjects(entries, requireDecryption) + ls.printDataObjects(entries) return nil } @@ -243,16 +230,16 @@ func (ls *LsCommand) printCollections(entries []*irodsclient_types.IRODSCollecti } } -func (ls *LsCommand) printDataObjects(entries []*irodsclient_types.IRODSDataObject, requireDecryption bool) { +func (ls *LsCommand) printDataObjects(entries []*irodsclient_types.IRODSDataObject) { if ls.listFlagValues.Format == commons.ListFormatNormal { sort.SliceStable(entries, ls.getDataObjectSortFunction(entries, ls.listFlagValues.SortOrder, ls.listFlagValues.SortReverse)) for _, entry := range entries { - ls.printDataObjectShort(entry, requireDecryption) + ls.printDataObjectShort(entry) } } else { replicas := ls.flattenReplicas(entries) sort.SliceStable(replicas, ls.getFlatReplicaSortFunction(replicas, ls.listFlagValues.SortOrder, ls.listFlagValues.SortReverse)) - ls.printReplicas(replicas, requireDecryption) + ls.printReplicas(replicas) } } @@ -405,7 +392,7 @@ func (ls *LsCommand) getDataObjectModifyTime(object *irodsclient_types.IRODSData return maxTime } -func (ls *LsCommand) printDataObjectShort(entry *irodsclient_types.IRODSDataObject, requireDecryption bool) { +func (ls *LsCommand) printDataObjectShort(entry *irodsclient_types.IRODSDataObject) { logger := log.WithFields(log.Fields{ "package": "subcmd", "struct": "LsCommand", @@ -414,7 +401,7 @@ func (ls *LsCommand) printDataObjectShort(entry *irodsclient_types.IRODSDataObje newName := entry.Name - if requireDecryption { + if ls.requireDecryption(entry.Path) { // need to decrypt encryptionMode := commons.DetectEncryptionMode(newName) if encryptionMode != commons.EncryptionModeUnknown { @@ -433,9 +420,9 @@ func (ls *LsCommand) printDataObjectShort(entry *irodsclient_types.IRODSDataObje fmt.Printf(" %s\n", newName) } -func (ls *LsCommand) printReplicas(flatReplicas []*FlatReplica, requireDecryption bool) { +func (ls *LsCommand) printReplicas(flatReplicas []*FlatReplica) { for _, flatReplica := range flatReplicas { - ls.printReplica(*flatReplica, requireDecryption) + ls.printReplica(*flatReplica) } } @@ -452,7 +439,7 @@ func (ls *LsCommand) getEncryptionManagerForDecryption(mode commons.EncryptionMo return manager } -func (ls *LsCommand) printReplica(flatReplica FlatReplica, requireDecryption bool) { +func (ls *LsCommand) printReplica(flatReplica FlatReplica) { logger := log.WithFields(log.Fields{ "package": "subcmd", "struct": "LsCommand", @@ -461,7 +448,7 @@ func (ls *LsCommand) printReplica(flatReplica FlatReplica, requireDecryption boo newName := flatReplica.DataObject.Name - if requireDecryption { + if ls.requireDecryption(flatReplica.DataObject.Path) { // need to decrypt encryptionMode := commons.DetectEncryptionMode(newName) if encryptionMode != commons.EncryptionModeUnknown { diff --git a/cmd/subcmd/mv.go b/cmd/subcmd/mv.go index c965940..b882af7 100644 --- a/cmd/subcmd/mv.go +++ b/cmd/subcmd/mv.go @@ -110,6 +110,11 @@ func (mv *MvCommand) ensureTargetIsDir(targetPath string) error { targetEntry, err := mv.filesystem.Stat(targetPath) if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // not exist + return commons.NewNotDirError(targetPath) + } + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } diff --git a/cmd/subcmd/put.go b/cmd/subcmd/put.go index e71f5fa..a915627 100644 --- a/cmd/subcmd/put.go +++ b/cmd/subcmd/put.go @@ -252,6 +252,11 @@ func (put *PutCommand) ensureTargetIsDir(targetPath string) error { targetEntry, err := put.filesystem.Stat(targetPath) if err != nil { + if irodsclient_types.IsFileNotFoundError(err) { + // not exist + return commons.NewNotDirError(targetPath) + } + return xerrors.Errorf("failed to stat %q: %w", targetPath, err) } diff --git a/commons/bundle_transfer.go b/commons/bundle_transfer.go index 0ed296d..6b8b7d7 100644 --- a/commons/bundle_transfer.go +++ b/commons/bundle_transfer.go @@ -357,7 +357,7 @@ func (manager *BundleTransferManager) Wait() error { } if manager.bundlesDoneCounter != manager.bundlesScheduledCounter { - return xerrors.Errorf("bundles '%d/%d' were canceled!", manager.bundlesDoneCounter, manager.bundlesScheduledCounter) + return xerrors.Errorf("bundles '%d/%d' were not completed!", manager.bundlesDoneCounter, manager.bundlesScheduledCounter) } return nil diff --git a/commons/parallel.go b/commons/parallel.go index d9eaf2b..ff44fa1 100644 --- a/commons/parallel.go +++ b/commons/parallel.go @@ -162,7 +162,7 @@ func (manager *ParallelJobManager) Wait() error { } if manager.jobsDoneCounter != manager.jobsScheduledCounter { - return xerrors.Errorf("jobs '%d/%d' were canceled!", manager.jobsDoneCounter, manager.jobsScheduledCounter) + return xerrors.Errorf("jobs '%d/%d' were not completed!", manager.jobsDoneCounter, manager.jobsScheduledCounter) } return nil @@ -294,12 +294,13 @@ func (manager *ParallelJobManager) Start() { currentThreads -= pjob.threadsRequired logger.Debugf("# threads : %d, max %d", currentThreads, manager.maxThreads) - manager.jobWait.Done() if pjob.done { // increase jobs done counter atomic.AddInt64(&manager.jobsDoneCounter, 1) } + manager.jobWait.Done() + manager.mutex.Lock() manager.availableThreadWaitCondition.Broadcast() manager.mutex.Unlock() From 8fd046532a22d66c69d3ba1aed603a3208a37d0c Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Tue, 20 Aug 2024 16:35:35 -0700 Subject: [PATCH 09/16] fix typo --- cmd/gocmd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/gocmd.go b/cmd/gocmd.go index e4e3e30..1387c33 100644 --- a/cmd/gocmd.go +++ b/cmd/gocmd.go @@ -175,9 +175,9 @@ func main() { } else if commons.IsNotDirError(err) { var notDirError *commons.NotDirError if errors.As(err, ¬DirError) { - fmt.Fprintf(os.Stderr, "Destination %q is not a director!\n", notDirError.Path) + fmt.Fprintf(os.Stderr, "Destination %q is not a directory!\n", notDirError.Path) } else { - fmt.Fprintf(os.Stderr, "Destination is not a director!\n") + fmt.Fprintf(os.Stderr, "Destination is not a directory!\n") } } else { fmt.Fprintf(os.Stderr, "Unexpected error!\nError Trace:\n - %+v\n", err) From bac8786f54a6ee8a3002588eb263b481621729de Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Wed, 21 Aug 2024 12:09:16 -0700 Subject: [PATCH 10/16] Do not print any messages in terminal when prompting input --- cmd/flag/common.go | 2 +- cmd/gocmd.go | 52 ++++++++++--------- cmd/subcmd/cat.go | 3 +- cmd/subcmd/ls.go | 14 ++--- cmd/subcmd/lsmeta.go | 18 +++---- cmd/subcmd/lsticket.go | 51 +++++++++--------- cmd/subcmd/passwd.go | 31 +++-------- cmd/subcmd/pwd.go | 4 +- cmd/subcmd/upgrade.go | 7 ++- commons/commands.go | 104 +++++++++++++++++++------------------ commons/input.go | 70 +++++++++++++++++++++++++ commons/print.go | 60 +++++++++++++++++++-- commons/retry.go | 5 +- commons/transfer_report.go | 3 +- 14 files changed, 263 insertions(+), 161 deletions(-) create mode 100644 commons/input.go diff --git a/cmd/flag/common.go b/cmd/flag/common.go index 1cba81c..66e601a 100644 --- a/cmd/flag/common.go +++ b/cmd/flag/common.go @@ -196,6 +196,6 @@ func printVersion() error { return xerrors.Errorf("failed to get version json: %w", err) } - fmt.Println(info) + commons.Println(info) return nil } diff --git a/cmd/gocmd.go b/cmd/gocmd.go index 1387c33..7838526 100644 --- a/cmd/gocmd.go +++ b/cmd/gocmd.go @@ -2,7 +2,6 @@ package main import ( "errors" - "fmt" "os" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" @@ -55,12 +54,15 @@ func processCommand(command *cobra.Command, args []string) error { } func main() { + commons.InitTerminalOutput() + log.SetFormatter(&log.TextFormatter{ TimestampFormat: "2006-01-02 15:04:05.000", FullTimestamp: true, }) log.SetLevel(log.FatalLevel) + log.SetOutput(commons.GetTerminalWriter()) logger := log.WithFields(log.Fields{ "package": "main", @@ -106,84 +108,84 @@ func main() { logger.Errorf("%+v", err) if os.IsNotExist(err) { - fmt.Fprintf(os.Stderr, "File or directory not found!\n") + commons.Fprintf(os.Stderr, "File or directory not found!\n") } else if irodsclient_types.IsConnectionConfigError(err) { var connectionConfigError *irodsclient_types.ConnectionConfigError if errors.As(err, &connectionConfigError) { - fmt.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server (host: %q, port: %d)!\n", connectionConfigError.Config.Host, connectionConfigError.Config.Port) + commons.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server (host: %q, port: %d)!\n", connectionConfigError.Config.Host, connectionConfigError.Config.Port) } else { - fmt.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server!\n") + commons.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server!\n") } } else if irodsclient_types.IsConnectionError(err) { - fmt.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server!\n") + commons.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server!\n") } else if irodsclient_types.IsConnectionPoolFullError(err) { var connectionPoolFullError *irodsclient_types.ConnectionPoolFullError if errors.As(err, &connectionPoolFullError) { - fmt.Fprintf(os.Stderr, "Failed to establish a new connection to iRODS server as connection pool is full (occupied: %d, max: %d)!\n", connectionPoolFullError.Occupied, connectionPoolFullError.Max) + commons.Fprintf(os.Stderr, "Failed to establish a new connection to iRODS server as connection pool is full (occupied: %d, max: %d)!\n", connectionPoolFullError.Occupied, connectionPoolFullError.Max) } else { - fmt.Fprintf(os.Stderr, "Failed to establish a new connection to iRODS server as connection pool is full!\n") + commons.Fprintf(os.Stderr, "Failed to establish a new connection to iRODS server as connection pool is full!\n") } } else if irodsclient_types.IsAuthError(err) { var authError *irodsclient_types.AuthError if errors.As(err, &authError) { - fmt.Fprintf(os.Stderr, "Authentication failed (auth scheme: %q, username: %q, zone: %q)!\n", authError.Config.AuthenticationScheme, authError.Config.ClientUser, authError.Config.ClientZone) + commons.Fprintf(os.Stderr, "Authentication failed (auth scheme: %q, username: %q, zone: %q)!\n", authError.Config.AuthenticationScheme, authError.Config.ClientUser, authError.Config.ClientZone) } else { - fmt.Fprintf(os.Stderr, "Authentication failed!\n") + commons.Fprintf(os.Stderr, "Authentication failed!\n") } } else if irodsclient_types.IsFileNotFoundError(err) { var fileNotFoundError *irodsclient_types.FileNotFoundError if errors.As(err, &fileNotFoundError) { - fmt.Fprintf(os.Stderr, "File or directory %q is not found!\n", fileNotFoundError.Path) + commons.Fprintf(os.Stderr, "File or directory %q is not found!\n", fileNotFoundError.Path) } else { - fmt.Fprintf(os.Stderr, "File or directory is not found!\n") + commons.Fprintf(os.Stderr, "File or directory is not found!\n") } } else if irodsclient_types.IsCollectionNotEmptyError(err) { var collectionNotEmptyError *irodsclient_types.CollectionNotEmptyError if errors.As(err, &collectionNotEmptyError) { - fmt.Fprintf(os.Stderr, "Directory %q is not empty!\n", collectionNotEmptyError.Path) + commons.Fprintf(os.Stderr, "Directory %q is not empty!\n", collectionNotEmptyError.Path) } else { - fmt.Fprintf(os.Stderr, "Directory is not empty!\n") + commons.Fprintf(os.Stderr, "Directory is not empty!\n") } } else if irodsclient_types.IsFileAlreadyExistError(err) { var fileAlreadyExistError *irodsclient_types.FileAlreadyExistError if errors.As(err, &fileAlreadyExistError) { - fmt.Fprintf(os.Stderr, "File or directory %q already exists!\n", fileAlreadyExistError.Path) + commons.Fprintf(os.Stderr, "File or directory %q already exists!\n", fileAlreadyExistError.Path) } else { - fmt.Fprintf(os.Stderr, "File or directory already exists!\n") + commons.Fprintf(os.Stderr, "File or directory already exists!\n") } } else if irodsclient_types.IsTicketNotFoundError(err) { var ticketNotFoundError *irodsclient_types.TicketNotFoundError if errors.As(err, &ticketNotFoundError) { - fmt.Fprintf(os.Stderr, "Ticket %q is not found!\n", ticketNotFoundError.Ticket) + commons.Fprintf(os.Stderr, "Ticket %q is not found!\n", ticketNotFoundError.Ticket) } else { - fmt.Fprintf(os.Stderr, "Ticket is not found!\n") + commons.Fprintf(os.Stderr, "Ticket is not found!\n") } } else if irodsclient_types.IsUserNotFoundError(err) { var userNotFoundError *irodsclient_types.UserNotFoundError if errors.As(err, &userNotFoundError) { - fmt.Fprintf(os.Stderr, "User %q is not found!\n", userNotFoundError.Name) + commons.Fprintf(os.Stderr, "User %q is not found!\n", userNotFoundError.Name) } else { - fmt.Fprintf(os.Stderr, "User is not found!\n") + commons.Fprintf(os.Stderr, "User is not found!\n") } } else if irodsclient_types.IsIRODSError(err) { var irodsError *irodsclient_types.IRODSError if errors.As(err, &irodsError) { - fmt.Fprintf(os.Stderr, "iRODS Error (code: '%d', message: %q)\n", irodsError.Code, irodsError.Error()) + commons.Fprintf(os.Stderr, "iRODS Error (code: '%d', message: %q)\n", irodsError.Code, irodsError.Error()) } else { - fmt.Fprintf(os.Stderr, "iRODS Error!\n") + commons.Fprintf(os.Stderr, "iRODS Error!\n") } } else if commons.IsNotDirError(err) { var notDirError *commons.NotDirError if errors.As(err, ¬DirError) { - fmt.Fprintf(os.Stderr, "Destination %q is not a directory!\n", notDirError.Path) + commons.Fprintf(os.Stderr, "Destination %q is not a directory!\n", notDirError.Path) } else { - fmt.Fprintf(os.Stderr, "Destination is not a directory!\n") + commons.Fprintf(os.Stderr, "Destination is not a directory!\n") } } else { - fmt.Fprintf(os.Stderr, "Unexpected error!\nError Trace:\n - %+v\n", err) + commons.Fprintf(os.Stderr, "Unexpected error!\nError Trace:\n - %+v\n", err) } - //fmt.Fprintf(os.Stderr, "\nError Trace:\n - %+v\n", err) + //commons.Fprintf(os.Stderr, "\nError Trace:\n - %+v\n", err) os.Exit(1) } } diff --git a/cmd/subcmd/cat.go b/cmd/subcmd/cat.go index 781a7a9..230e4ed 100644 --- a/cmd/subcmd/cat.go +++ b/cmd/subcmd/cat.go @@ -1,7 +1,6 @@ package subcmd import ( - "fmt" "io" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" @@ -147,7 +146,7 @@ func (cat *CatCommand) catOne(sourcePath string) error { for { readLen, err := fh.Read(buf) if readLen > 0 { - fmt.Printf("%s", string(buf[:readLen])) + commons.Printf("%s", string(buf[:readLen])) } if err == io.EOF { diff --git a/cmd/subcmd/ls.go b/cmd/subcmd/ls.go index 87f8fa6..2471080 100644 --- a/cmd/subcmd/ls.go +++ b/cmd/subcmd/ls.go @@ -226,7 +226,7 @@ func (ls *LsCommand) listOne(sourcePath string) error { func (ls *LsCommand) printCollections(entries []*irodsclient_types.IRODSCollection) { sort.SliceStable(entries, ls.getCollectionSortFunction(entries, ls.listFlagValues.SortOrder, ls.listFlagValues.SortReverse)) for _, entry := range entries { - fmt.Printf(" C- %s\n", entry.Path) + commons.Printf(" C- %s\n", entry.Path) } } @@ -417,7 +417,7 @@ func (ls *LsCommand) printDataObjectShort(entry *irodsclient_types.IRODSDataObje } } - fmt.Printf(" %s\n", newName) + commons.Printf(" %s\n", newName) } func (ls *LsCommand) printReplicas(flatReplicas []*FlatReplica) { @@ -471,18 +471,18 @@ func (ls *LsCommand) printReplica(flatReplica FlatReplica) { switch ls.listFlagValues.Format { case commons.ListFormatNormal: - fmt.Printf(" %d\t%s\n", flatReplica.Replica.Number, newName) + commons.Printf(" %d\t%s\n", flatReplica.Replica.Number, newName) case commons.ListFormatLong: modTime := commons.MakeDateTimeString(flatReplica.Replica.ModifyTime) - fmt.Printf(" %s\t%d\t%s\t%s\t%s\t%s\t%s\n", flatReplica.Replica.Owner, flatReplica.Replica.Number, flatReplica.Replica.ResourceHierarchy, + commons.Printf(" %s\t%d\t%s\t%s\t%s\t%s\t%s\n", flatReplica.Replica.Owner, flatReplica.Replica.Number, flatReplica.Replica.ResourceHierarchy, size, modTime, ls.getStatusMark(flatReplica.Replica.Status), newName) case commons.ListFormatVeryLong: modTime := commons.MakeDateTimeString(flatReplica.Replica.ModifyTime) - fmt.Printf(" %s\t%d\t%s\t%s\t%s\t%s\t%s\n", flatReplica.Replica.Owner, flatReplica.Replica.Number, flatReplica.Replica.ResourceHierarchy, + commons.Printf(" %s\t%d\t%s\t%s\t%s\t%s\t%s\n", flatReplica.Replica.Owner, flatReplica.Replica.Number, flatReplica.Replica.ResourceHierarchy, size, modTime, ls.getStatusMark(flatReplica.Replica.Status), newName) - fmt.Printf(" %s\t%s\n", flatReplica.Replica.Checksum.IRODSChecksumString, flatReplica.Replica.Path) + commons.Printf(" %s\t%s\n", flatReplica.Replica.Checksum.IRODSChecksumString, flatReplica.Replica.Path) default: - fmt.Printf(" %d\t%s\n", flatReplica.Replica.Number, newName) + commons.Printf(" %d\t%s\n", flatReplica.Replica.Number, newName) } } diff --git a/cmd/subcmd/lsmeta.go b/cmd/subcmd/lsmeta.go index e6dc057..ba414b5 100644 --- a/cmd/subcmd/lsmeta.go +++ b/cmd/subcmd/lsmeta.go @@ -182,18 +182,18 @@ func (lsMeta *LsMetaCommand) printMetaInternal(meta *types.IRODSMeta) { switch lsMeta.listFlagValues.Format { case commons.ListFormatLong, commons.ListFormatVeryLong: - fmt.Printf("[%s]\n", meta.Name) - fmt.Printf("[%s]\n", meta.Name) - fmt.Printf(" id: %d\n", meta.AVUID) - fmt.Printf(" attribute: %s\n", name) - fmt.Printf(" value: %s\n", value) - fmt.Printf(" unit: %s\n", units) - fmt.Printf(" create time: %s\n", createTime) - fmt.Printf(" modify time: %s\n", modTime) + commons.Printf("[%s]\n", meta.Name) + commons.Printf("[%s]\n", meta.Name) + commons.Printf(" id: %d\n", meta.AVUID) + commons.Printf(" attribute: %s\n", name) + commons.Printf(" value: %s\n", value) + commons.Printf(" unit: %s\n", units) + commons.Printf(" create time: %s\n", createTime) + commons.Printf(" modify time: %s\n", modTime) case commons.ListFormatNormal: fallthrough default: - fmt.Printf("%d\t%s\t%s\t%s\n", meta.AVUID, name, value, units) + commons.Printf("%d\t%s\t%s\t%s\n", meta.AVUID, name, value, units) } } diff --git a/cmd/subcmd/lsticket.go b/cmd/subcmd/lsticket.go index 0a4f100..55d46ee 100644 --- a/cmd/subcmd/lsticket.go +++ b/cmd/subcmd/lsticket.go @@ -1,7 +1,6 @@ package subcmd import ( - "fmt" "sort" irodsclient_fs "github.com/cyverse/go-irodsclient/fs" @@ -148,25 +147,25 @@ func (lsTicket *LsTicketCommand) printTickets(tickets []*types.IRODSTicket) erro } func (lsTicket *LsTicketCommand) printTicketInternal(ticket *types.IRODSTicket) error { - fmt.Printf("[%s]\n", ticket.Name) - fmt.Printf(" id: %d\n", ticket.ID) - fmt.Printf(" name: %s\n", ticket.Name) - fmt.Printf(" type: %s\n", ticket.Type) - fmt.Printf(" owner: %s\n", ticket.Owner) - fmt.Printf(" owner zone: %s\n", ticket.OwnerZone) - fmt.Printf(" object type: %s\n", ticket.ObjectType) - fmt.Printf(" path: %s\n", ticket.Path) - fmt.Printf(" uses limit: %d\n", ticket.UsesLimit) - fmt.Printf(" uses count: %d\n", ticket.UsesCount) - fmt.Printf(" write file limit: %d\n", ticket.WriteFileLimit) - fmt.Printf(" write file count: %d\n", ticket.WriteFileCount) - fmt.Printf(" write byte limit: %d\n", ticket.WriteByteLimit) - fmt.Printf(" write byte count: %d\n", ticket.WriteByteCount) + commons.Printf("[%s]\n", ticket.Name) + commons.Printf(" id: %d\n", ticket.ID) + commons.Printf(" name: %s\n", ticket.Name) + commons.Printf(" type: %s\n", ticket.Type) + commons.Printf(" owner: %s\n", ticket.Owner) + commons.Printf(" owner zone: %s\n", ticket.OwnerZone) + commons.Printf(" object type: %s\n", ticket.ObjectType) + commons.Printf(" path: %s\n", ticket.Path) + commons.Printf(" uses limit: %d\n", ticket.UsesLimit) + commons.Printf(" uses count: %d\n", ticket.UsesCount) + commons.Printf(" write file limit: %d\n", ticket.WriteFileLimit) + commons.Printf(" write file count: %d\n", ticket.WriteFileCount) + commons.Printf(" write byte limit: %d\n", ticket.WriteByteLimit) + commons.Printf(" write byte count: %d\n", ticket.WriteByteCount) if ticket.ExpirationTime.IsZero() { - fmt.Print(" expiry time: none\n") + commons.Print(" expiry time: none\n") } else { - fmt.Printf(" expiry time: %s\n", commons.MakeDateTimeString(ticket.ExpirationTime)) + commons.Printf(" expiry time: %s\n", commons.MakeDateTimeString(ticket.ExpirationTime)) } if lsTicket.listFlagValues.Format == commons.ListFormatLong || lsTicket.listFlagValues.Format == commons.ListFormatVeryLong { @@ -177,29 +176,29 @@ func (lsTicket *LsTicketCommand) printTicketInternal(ticket *types.IRODSTicket) if restrictions != nil { if len(restrictions.AllowedHosts) == 0 { - fmt.Printf(" No host restrictions\n") + commons.Printf(" No host restrictions\n") } else { for _, host := range restrictions.AllowedHosts { - fmt.Printf(" Allowed Hosts:\n") - fmt.Printf(" - %s\n", host) + commons.Printf(" Allowed Hosts:\n") + commons.Printf(" - %s\n", host) } } if len(restrictions.AllowedUserNames) == 0 { - fmt.Printf(" No user restrictions\n") + commons.Printf(" No user restrictions\n") } else { for _, user := range restrictions.AllowedUserNames { - fmt.Printf(" Allowed Users:\n") - fmt.Printf(" - %s\n", user) + commons.Printf(" Allowed Users:\n") + commons.Printf(" - %s\n", user) } } if len(restrictions.AllowedGroupNames) == 0 { - fmt.Printf(" No group restrictions\n") + commons.Printf(" No group restrictions\n") } else { for _, group := range restrictions.AllowedGroupNames { - fmt.Printf(" Allowed Groups:\n") - fmt.Printf(" - %s\n", group) + commons.Printf(" Allowed Groups:\n") + commons.Printf(" - %s\n", group) } } } diff --git a/cmd/subcmd/passwd.go b/cmd/subcmd/passwd.go index 16489be..e2df00f 100644 --- a/cmd/subcmd/passwd.go +++ b/cmd/subcmd/passwd.go @@ -1,9 +1,6 @@ package subcmd import ( - "fmt" - "syscall" - irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_irodsfs "github.com/cyverse/go-irodsclient/irods/fs" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" @@ -11,7 +8,6 @@ import ( "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "golang.org/x/term" "golang.org/x/xerrors" ) @@ -102,22 +98,18 @@ func (passwd *PasswdCommand) changePassword() error { pass := false for i := 0; i < 3; i++ { - fmt.Print("Current iRODS Password: ") - bytePassword, err := term.ReadPassword(int(syscall.Stdin)) + currentPassword, err := commons.InputPassword("Current iRODS Password") if err != nil { return xerrors.Errorf("failed to read password: %w", err) } - fmt.Print("\n") - currentPassword := string(bytePassword) - if currentPassword == passwd.account.Password { pass = true break } - fmt.Println("Wrong password") - fmt.Println("") + commons.Println("Wrong password") + commons.Println("") } if !pass { @@ -127,38 +119,29 @@ func (passwd *PasswdCommand) changePassword() error { pass = false newPassword := "" for i := 0; i < 3; i++ { - fmt.Print("New iRODS Password: ") - bytePassword, err := term.ReadPassword(int(syscall.Stdin)) + newPassword, err = commons.InputPassword("New iRODS Password") if err != nil { return xerrors.Errorf("failed to read password: %w", err) } - fmt.Print("\n") - newPassword = string(bytePassword) - if newPassword != passwd.account.Password { pass = true break } - fmt.Println("Please provide new password") - fmt.Println("") + commons.Println("Please provide new password") + commons.Println("") } if !pass { return xerrors.Errorf("invalid password provided") } - newPasswordConfirm := "" - fmt.Print("Confirm New iRODS Password: ") - bytePassword, err := term.ReadPassword(int(syscall.Stdin)) + newPasswordConfirm, err := commons.InputPassword("Confirm New iRODS Password") if err != nil { return xerrors.Errorf("failed to read password: %w", err) } - fmt.Print("\n") - newPasswordConfirm = string(bytePassword) - if newPassword != newPasswordConfirm { return xerrors.Errorf("password mismatched") } diff --git a/cmd/subcmd/pwd.go b/cmd/subcmd/pwd.go index 5b1482e..23e84d8 100644 --- a/cmd/subcmd/pwd.go +++ b/cmd/subcmd/pwd.go @@ -1,8 +1,6 @@ package subcmd import ( - "fmt" - irodsclient_fs "github.com/cyverse/go-irodsclient/fs" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/gocommands/cmd/flag" @@ -76,7 +74,7 @@ func (pwd *PwdCommand) Process() error { func (pwd *PwdCommand) printCurrentWorkingDir() error { cwd := commons.GetCWD() - fmt.Printf("%s\n", cwd) + commons.Printf("%s\n", cwd) return nil } diff --git a/cmd/subcmd/upgrade.go b/cmd/subcmd/upgrade.go index 9534386..81494fa 100644 --- a/cmd/subcmd/upgrade.go +++ b/cmd/subcmd/upgrade.go @@ -1,7 +1,6 @@ package subcmd import ( - "fmt" "runtime" "github.com/cyverse/gocommands/cmd/flag" @@ -85,11 +84,11 @@ func (upgrade *UpgradeCommand) checkNewVersion() error { return err } - fmt.Printf("Latest version v%s for %s/%s\n", newRelease.Version(), runtime.GOOS, runtime.GOARCH) - fmt.Printf("Latest release URL: %s\n", newRelease.URL) + commons.Printf("Latest version v%s for %s/%s\n", newRelease.Version(), runtime.GOOS, runtime.GOARCH) + commons.Printf("Latest release URL: %s\n", newRelease.URL) myVersion := commons.GetClientVersion() - fmt.Printf("Current cilent version installed: %s\n", myVersion) + commons.Printf("Current cilent version installed: %s\n", myVersion) return nil } diff --git a/commons/commands.go b/commons/commands.go index ca960e7..f14648f 100644 --- a/commons/commands.go +++ b/commons/commands.go @@ -7,13 +7,11 @@ import ( "path" "path/filepath" "strings" - "syscall" irodsclient_icommands "github.com/cyverse/go-irodsclient/icommands" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "github.com/cyverse/go-irodsclient/irods/util" log "github.com/sirupsen/logrus" - "golang.org/x/term" "golang.org/x/xerrors" "github.com/jedib0t/go-pretty/v6/table" @@ -189,14 +187,22 @@ func InputMissingFields() (bool, error) { env := environmentManager.Environment if len(env.Host) == 0 { - fmt.Print("iRODS Host [data.cyverse.org]: ") - fmt.Scanln(&env.Host) + host, err := Input("iRODS Host [data.cyverse.org]") + if err != nil { + return false, xerrors.Errorf("failed to read input: %w", err) + } + + env.Host = host if len(env.Host) == 0 { env.Host = "data.cyverse.org" } - fmt.Print("iRODS Port [1247]: ") - fmt.Scanln(&env.Port) + port, err := InputInt("iRODS Port [1247]") + if err != nil { + return false, xerrors.Errorf("failed to read input: %w", err) + } + + env.Port = port if env.Port == 0 { env.Port = 1247 } @@ -205,8 +211,12 @@ func InputMissingFields() (bool, error) { } if len(env.Zone) == 0 { - fmt.Print("iRODS Zone [iplant]: ") - fmt.Scanln(&env.Zone) + zone, err := Input("iRODS Zone [iplant]") + if err != nil { + return false, xerrors.Errorf("failed to read input: %w", err) + } + + env.Zone = zone if len(env.Zone) == 0 { env.Zone = "iplant" } @@ -215,25 +225,27 @@ func InputMissingFields() (bool, error) { } if len(env.Username) == 0 { - fmt.Print("iRODS Username: ") - fmt.Scanln(&env.Username) + username, err := Input("iRODS Username") + if err != nil { + return false, xerrors.Errorf("failed to read input: %w", err) + } + + env.Username = username updated = true } password := environmentManager.Password pamToken := environmentManager.PamToken if len(password) == 0 && len(pamToken) == 0 && env.Username != "anonymous" { - fmt.Print("iRODS Password: ") - bytePassword, err := term.ReadPassword(int(syscall.Stdin)) + password, err := InputPassword("iRODS Password") if err != nil { return false, xerrors.Errorf("failed to read password: %w", err) } - fmt.Print("\n") - password = string(bytePassword) + environmentManager.Password = password updated = true } - environmentManager.Password = password + err := SyncAccount() if err != nil { return updated, xerrors.Errorf("failed to get iCommands Environment: %w", err) @@ -296,9 +308,11 @@ func ReinputFields() (bool, error) { env.Host = "data.cyverse.org" // default } - fmt.Printf("iRODS Host [%s]: ", env.Host) - newHost := "" - fmt.Scanln(&newHost) + newHost, err := Input(fmt.Sprintf("iRODS Host [%s]", env.Host)) + if err != nil { + return false, xerrors.Errorf("failed to read input: %w", err) + } + if len(newHost) > 0 && newHost != env.Host { env.Host = newHost updated = true @@ -308,9 +322,11 @@ func ReinputFields() (bool, error) { env.Port = 1247 // default } - fmt.Printf("iRODS Port [%d]: ", env.Port) - newPort := 0 - fmt.Scanln(&newPort) + newPort, err := InputInt(fmt.Sprintf("iRODS Port [%d]", env.Port)) + if err != nil { + return false, xerrors.Errorf("failed to read input: %w", err) + } + if newPort > 0 && newPort != env.Port { env.Port = newPort updated = true @@ -320,23 +336,30 @@ func ReinputFields() (bool, error) { env.Zone = "iplant" // default } - fmt.Printf("iRODS Zone [%s]: ", env.Zone) - newZone := "" - fmt.Scanln(&newZone) + newZone, err := Input(fmt.Sprintf("iRODS Zone [%s]", env.Zone)) + if err != nil { + return false, xerrors.Errorf("failed to read input: %w", err) + } + if len(newZone) > 0 && newZone != env.Zone { env.Zone = newZone updated = true } for { + newUsername := "" if len(env.Username) > 0 { - fmt.Printf("iRODS Username [%s]: ", env.Username) + newUsername, err = Input(fmt.Sprintf("iRODS Username [%s]", env.Username)) + if err != nil { + return false, xerrors.Errorf("failed to read input: %w", err) + } } else { - fmt.Printf("iRODS Username: ") + newUsername, err = Input("iRODS Username") + if err != nil { + return false, xerrors.Errorf("failed to read input: %w", err) + } } - newUsername := "" - fmt.Scanln(&newUsername) if len(newUsername) > 0 && newUsername != env.Username { env.Username = newUsername updated = true @@ -347,13 +370,11 @@ func ReinputFields() (bool, error) { } } - fmt.Print("iRODS Password: ") - bytePassword, err := term.ReadPassword(int(syscall.Stdin)) + newPassword, err := InputPassword("iRODS Password") if err != nil { return false, xerrors.Errorf("failed to read password: %w", err) } - fmt.Print("\n") - newPassword := string(bytePassword) + updated = true environmentManager.Password = newPassword @@ -743,22 +764,3 @@ func PrintAccount() error { t.Render() return nil } - -// InputYN inputs Y or N -// true for Y, false for N -func InputYN(msg string) bool { - userInput := "" - - for { - fmt.Printf("%s [y/n]: ", msg) - - fmt.Scanln(&userInput) - userInput = strings.ToLower(userInput) - - if userInput == "y" || userInput == "yes" { - return true - } else if userInput == "n" || userInput == "no" { - return false - } - } -} diff --git a/commons/input.go b/commons/input.go new file mode 100644 index 0000000..46bc98e --- /dev/null +++ b/commons/input.go @@ -0,0 +1,70 @@ +package commons + +import ( + "fmt" + "strconv" + "strings" + "syscall" + + "golang.org/x/term" +) + +func Input(msg string) (string, error) { + terminalWriter := GetTerminalWriter() + + terminalWriter.Lock() + defer terminalWriter.Unlock() + + red := "\033[31m" + reset := "\033[0m" + + fmt.Printf("%s%s: %s", red, msg, reset) + + userInput := "" + _, err := fmt.Scanln(&userInput) + + return userInput, err +} + +// InputYN inputs Y or N +// true for Y, false for N +func InputYN(msg string) bool { + for { + inputString, _ := Input(msg) + inputString = strings.ToLower(inputString) + if inputString == "y" || inputString == "yes" || inputString == "true" { + return true + } else if inputString == "n" || inputString == "no" || inputString == "false" { + return false + } + } +} + +func InputInt(msg string) (int, error) { + inputString, err := Input(msg) + if err != nil { + return -1, err + } + + return strconv.Atoi(inputString) +} + +func InputPassword(msg string) (string, error) { + terminalWriter := GetTerminalWriter() + + terminalWriter.Lock() + defer terminalWriter.Unlock() + + red := "\033[31m" + reset := "\033[0m" + + fmt.Printf("%s%s: %s", red, msg, reset) + bytePassword, err := term.ReadPassword(int(syscall.Stdin)) + fmt.Print("\n") + + if err != nil { + return "", err + } + + return string(bytePassword), nil +} diff --git a/commons/print.go b/commons/print.go index dc1b891..0757232 100644 --- a/commons/print.go +++ b/commons/print.go @@ -2,20 +2,72 @@ package commons import ( "fmt" + "io" + "os" + "sync" log "github.com/sirupsen/logrus" ) -func Println(a ...any) (n int, err error) { +var ( + terminalOutput *TerminalWriter +) + +type TerminalWriter struct { + mutex sync.Mutex +} + +func (writer *TerminalWriter) Write(p []byte) (n int, err error) { + writer.mutex.Lock() + defer writer.mutex.Unlock() + return os.Stdout.Write(p) +} + +func (writer *TerminalWriter) Lock() { + writer.mutex.Lock() +} + +func (writer *TerminalWriter) Unlock() { + writer.mutex.Unlock() +} + +func InitTerminalOutput() { + terminalOutput = &TerminalWriter{} +} + +func GetTerminalWriter() *TerminalWriter { + return terminalOutput +} + +func PrintInfoln(a ...any) (n int, err error) { if log.GetLevel() > log.InfoLevel { - return fmt.Println(a...) + return Println(a...) } return 0, nil } -func Printf(format string, a ...any) (n int, err error) { +func PrintInfof(format string, a ...any) (n int, err error) { if log.GetLevel() > log.InfoLevel { - return fmt.Printf(format, a...) + return Printf(format, a...) } return 0, nil } + +func Print(a ...any) (n int, err error) { + return fmt.Fprint(terminalOutput, a...) +} + +func Printf(format string, a ...any) (n int, err error) { + return fmt.Fprintf(terminalOutput, format, a...) +} + +func Println(a ...any) (n int, err error) { + return fmt.Fprintln(terminalOutput, a...) +} + +func Fprintf(w io.Writer, format string, a ...any) (n int, err error) { + terminalOutput.Lock() + defer terminalOutput.Unlock() + + return fmt.Fprintf(w, format, a...) +} diff --git a/commons/retry.go b/commons/retry.go index 9a132b9..7c548d7 100644 --- a/commons/retry.go +++ b/commons/retry.go @@ -1,7 +1,6 @@ package commons import ( - "fmt" "os" "os/exec" "strings" @@ -27,10 +26,10 @@ func RunWithRetry(retry int, retryInterval int) error { } logger.Errorf("%+v", err) - fmt.Fprintf(os.Stderr, "Error: %+v\n", err) + Fprintf(os.Stderr, "Error: %+v\n", err) logger.Errorf("Waiting %d seconds for next try...", retryInterval) - fmt.Fprintf(os.Stderr, "Waiting %d seconds for next try...", retryInterval) + Fprintf(os.Stderr, "Waiting %d seconds for next try...", retryInterval) sleepTime := time.Duration(retryInterval * int(time.Second)) time.Sleep(sleepTime) diff --git a/commons/transfer_report.go b/commons/transfer_report.go index 51c59f6..77fbf00 100644 --- a/commons/transfer_report.go +++ b/commons/transfer_report.go @@ -3,7 +3,6 @@ package commons import ( "encoding/hex" "encoding/json" - "fmt" "io" "os" "strings" @@ -179,7 +178,7 @@ func (manager *TransferReportManager) AddFile(file *TransferReportFile) error { lineOutput := "" if manager.reportToStdout { // line print - fmt.Printf("[%s]\t%s\t%s\t%s\t%d\t%s\t%s\t%d\t%s\n", file.Method, file.StartAt, file.EndAt, file.SourcePath, file.SourceSize, file.SourceChecksum, file.DestPath, file.DestSize, file.DestChecksum) + Printf("[%s]\t%s\t%s\t%s\t%d\t%s\t%s\t%d\t%s\n", file.Method, file.StartAt, file.EndAt, file.SourcePath, file.SourceSize, file.SourceChecksum, file.DestPath, file.DestSize, file.DestChecksum) } else { // json fileBytes, err := json.Marshal(file) From 33f60ad0df0ba444b52b25b3a30d7a37fe81d3c1 Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Wed, 21 Aug 2024 15:02:40 -0700 Subject: [PATCH 11/16] bugfix: mark done for parallel jobs --- cmd/subcmd/cp.go | 2 ++ cmd/subcmd/get.go | 1 - cmd/subcmd/put.go | 1 + commons/input.go | 2 +- commons/parallel.go | 2 -- 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/subcmd/cp.go b/cmd/subcmd/cp.go index 4500ee9..f656402 100644 --- a/cmd/subcmd/cp.go +++ b/cmd/subcmd/cp.go @@ -284,6 +284,8 @@ func (cp *CpCommand) scheduleCopy(sourceEntry *irodsclient_fs.Entry, targetPath logger.Debugf("copied a data object %q to %q", sourceEntry.Path, targetPath) job.Progress(1, 1, false) + + job.Done() return nil } diff --git a/cmd/subcmd/get.go b/cmd/subcmd/get.go index 6bedb7e..d0f5a76 100644 --- a/cmd/subcmd/get.go +++ b/cmd/subcmd/get.go @@ -407,7 +407,6 @@ func (get *GetCommand) scheduleGet(sourceEntry *irodsclient_fs.Entry, tempPath s job.Progress(sourceEntry.Size, sourceEntry.Size, false) job.Done() - return nil } diff --git a/cmd/subcmd/put.go b/cmd/subcmd/put.go index a915627..1718aad 100644 --- a/cmd/subcmd/put.go +++ b/cmd/subcmd/put.go @@ -432,6 +432,7 @@ func (put *PutCommand) schedulePut(sourceStat fs.FileInfo, sourcePath string, te logger.Debugf("uploaded a file %q to %q", sourcePath, targetPath) job.Progress(sourceStat.Size(), sourceStat.Size(), false) + job.Done() return nil } diff --git a/commons/input.go b/commons/input.go index 46bc98e..d2a4cd1 100644 --- a/commons/input.go +++ b/commons/input.go @@ -30,7 +30,7 @@ func Input(msg string) (string, error) { // true for Y, false for N func InputYN(msg string) bool { for { - inputString, _ := Input(msg) + inputString, _ := Input(fmt.Sprintf("%s [y/n]", msg)) inputString = strings.ToLower(inputString) if inputString == "y" || inputString == "yes" || inputString == "true" { return true diff --git a/commons/parallel.go b/commons/parallel.go index ff44fa1..26910c8 100644 --- a/commons/parallel.go +++ b/commons/parallel.go @@ -20,7 +20,6 @@ type ParallelJob struct { task ParallelJobTask threadsRequired int progressUnit progress.Units - lastError error done bool } @@ -45,7 +44,6 @@ func newParallelJob(manager *ParallelJobManager, index int64, name string, task task: task, threadsRequired: threadsRequired, progressUnit: progressUnit, - lastError: nil, done: false, } From 58437ee00fd8a8184d62051307c3152bdfdc3a1d Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Wed, 21 Aug 2024 16:36:25 -0700 Subject: [PATCH 12/16] handle empty input --- cmd/subcmd/passwd.go | 18 ++--------- commons/commands.go | 71 ++++++++------------------------------------ commons/input.go | 27 ++++++++++------- 3 files changed, 31 insertions(+), 85 deletions(-) diff --git a/cmd/subcmd/passwd.go b/cmd/subcmd/passwd.go index e2df00f..70be7af 100644 --- a/cmd/subcmd/passwd.go +++ b/cmd/subcmd/passwd.go @@ -98,11 +98,7 @@ func (passwd *PasswdCommand) changePassword() error { pass := false for i := 0; i < 3; i++ { - currentPassword, err := commons.InputPassword("Current iRODS Password") - if err != nil { - return xerrors.Errorf("failed to read password: %w", err) - } - + currentPassword := commons.InputPassword("Current iRODS Password") if currentPassword == passwd.account.Password { pass = true break @@ -119,11 +115,7 @@ func (passwd *PasswdCommand) changePassword() error { pass = false newPassword := "" for i := 0; i < 3; i++ { - newPassword, err = commons.InputPassword("New iRODS Password") - if err != nil { - return xerrors.Errorf("failed to read password: %w", err) - } - + newPassword = commons.InputPassword("New iRODS Password") if newPassword != passwd.account.Password { pass = true break @@ -137,11 +129,7 @@ func (passwd *PasswdCommand) changePassword() error { return xerrors.Errorf("invalid password provided") } - newPasswordConfirm, err := commons.InputPassword("Confirm New iRODS Password") - if err != nil { - return xerrors.Errorf("failed to read password: %w", err) - } - + newPasswordConfirm := commons.InputPassword("Confirm New iRODS Password") if newPassword != newPasswordConfirm { return xerrors.Errorf("password mismatched") } diff --git a/commons/commands.go b/commons/commands.go index f14648f..954e91f 100644 --- a/commons/commands.go +++ b/commons/commands.go @@ -187,22 +187,12 @@ func InputMissingFields() (bool, error) { env := environmentManager.Environment if len(env.Host) == 0 { - host, err := Input("iRODS Host [data.cyverse.org]") - if err != nil { - return false, xerrors.Errorf("failed to read input: %w", err) - } - - env.Host = host + env.Host = Input("iRODS Host [data.cyverse.org]") if len(env.Host) == 0 { env.Host = "data.cyverse.org" } - port, err := InputInt("iRODS Port [1247]") - if err != nil { - return false, xerrors.Errorf("failed to read input: %w", err) - } - - env.Port = port + env.Port = InputInt("iRODS Port [1247]") if env.Port == 0 { env.Port = 1247 } @@ -211,12 +201,7 @@ func InputMissingFields() (bool, error) { } if len(env.Zone) == 0 { - zone, err := Input("iRODS Zone [iplant]") - if err != nil { - return false, xerrors.Errorf("failed to read input: %w", err) - } - - env.Zone = zone + env.Zone = Input("iRODS Zone [iplant]") if len(env.Zone) == 0 { env.Zone = "iplant" } @@ -225,24 +210,14 @@ func InputMissingFields() (bool, error) { } if len(env.Username) == 0 { - username, err := Input("iRODS Username") - if err != nil { - return false, xerrors.Errorf("failed to read input: %w", err) - } - - env.Username = username + env.Username = Input("iRODS Username") updated = true } password := environmentManager.Password pamToken := environmentManager.PamToken if len(password) == 0 && len(pamToken) == 0 && env.Username != "anonymous" { - password, err := InputPassword("iRODS Password") - if err != nil { - return false, xerrors.Errorf("failed to read password: %w", err) - } - - environmentManager.Password = password + environmentManager.Password = InputPassword("iRODS Password") updated = true } @@ -308,11 +283,7 @@ func ReinputFields() (bool, error) { env.Host = "data.cyverse.org" // default } - newHost, err := Input(fmt.Sprintf("iRODS Host [%s]", env.Host)) - if err != nil { - return false, xerrors.Errorf("failed to read input: %w", err) - } - + newHost := Input(fmt.Sprintf("iRODS Host [%s]", env.Host)) if len(newHost) > 0 && newHost != env.Host { env.Host = newHost updated = true @@ -322,11 +293,7 @@ func ReinputFields() (bool, error) { env.Port = 1247 // default } - newPort, err := InputInt(fmt.Sprintf("iRODS Port [%d]", env.Port)) - if err != nil { - return false, xerrors.Errorf("failed to read input: %w", err) - } - + newPort := InputInt(fmt.Sprintf("iRODS Port [%d]", env.Port)) if newPort > 0 && newPort != env.Port { env.Port = newPort updated = true @@ -336,11 +303,7 @@ func ReinputFields() (bool, error) { env.Zone = "iplant" // default } - newZone, err := Input(fmt.Sprintf("iRODS Zone [%s]", env.Zone)) - if err != nil { - return false, xerrors.Errorf("failed to read input: %w", err) - } - + newZone := Input(fmt.Sprintf("iRODS Zone [%s]", env.Zone)) if len(newZone) > 0 && newZone != env.Zone { env.Zone = newZone updated = true @@ -349,15 +312,9 @@ func ReinputFields() (bool, error) { for { newUsername := "" if len(env.Username) > 0 { - newUsername, err = Input(fmt.Sprintf("iRODS Username [%s]", env.Username)) - if err != nil { - return false, xerrors.Errorf("failed to read input: %w", err) - } + newUsername = Input(fmt.Sprintf("iRODS Username [%s]", env.Username)) } else { - newUsername, err = Input("iRODS Username") - if err != nil { - return false, xerrors.Errorf("failed to read input: %w", err) - } + newUsername = Input("iRODS Username") } if len(newUsername) > 0 && newUsername != env.Username { @@ -370,16 +327,12 @@ func ReinputFields() (bool, error) { } } - newPassword, err := InputPassword("iRODS Password") - if err != nil { - return false, xerrors.Errorf("failed to read password: %w", err) - } - + newPassword := InputPassword("iRODS Password") updated = true environmentManager.Password = newPassword - err = SyncAccount() + err := SyncAccount() if err != nil { return updated, xerrors.Errorf("failed to get iCommands Environment: %w", err) } diff --git a/commons/input.go b/commons/input.go index d2a4cd1..06e94b6 100644 --- a/commons/input.go +++ b/commons/input.go @@ -9,7 +9,7 @@ import ( "golang.org/x/term" ) -func Input(msg string) (string, error) { +func Input(msg string) string { terminalWriter := GetTerminalWriter() terminalWriter.Lock() @@ -21,16 +21,16 @@ func Input(msg string) (string, error) { fmt.Printf("%s%s: %s", red, msg, reset) userInput := "" - _, err := fmt.Scanln(&userInput) + fmt.Scanln(&userInput) - return userInput, err + return userInput } // InputYN inputs Y or N // true for Y, false for N func InputYN(msg string) bool { for { - inputString, _ := Input(fmt.Sprintf("%s [y/n]", msg)) + inputString := Input(fmt.Sprintf("%s [y/n]", msg)) inputString = strings.ToLower(inputString) if inputString == "y" || inputString == "yes" || inputString == "true" { return true @@ -40,16 +40,21 @@ func InputYN(msg string) bool { } } -func InputInt(msg string) (int, error) { - inputString, err := Input(msg) +func InputInt(msg string) int { + inputString := Input(msg) + if len(inputString) == 0 { + return 0 + } + + v, err := strconv.Atoi(inputString) if err != nil { - return -1, err + return 0 } - return strconv.Atoi(inputString) + return v } -func InputPassword(msg string) (string, error) { +func InputPassword(msg string) string { terminalWriter := GetTerminalWriter() terminalWriter.Lock() @@ -63,8 +68,8 @@ func InputPassword(msg string) (string, error) { fmt.Print("\n") if err != nil { - return "", err + return "" } - return string(bytePassword), nil + return string(bytePassword) } From a27c837f560650e35cd935d7c68e24ff0d862bdf Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Wed, 21 Aug 2024 17:12:16 -0700 Subject: [PATCH 13/16] Color error message in red --- cmd/gocmd.go | 47 +++++++++++++++++++++++------------------------ commons/print.go | 16 +++++++++++++++- commons/retry.go | 4 ++-- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/cmd/gocmd.go b/cmd/gocmd.go index 7838526..2336f87 100644 --- a/cmd/gocmd.go +++ b/cmd/gocmd.go @@ -108,84 +108,83 @@ func main() { logger.Errorf("%+v", err) if os.IsNotExist(err) { - commons.Fprintf(os.Stderr, "File or directory not found!\n") + commons.PrintErrorf("File or directory not found!\n") } else if irodsclient_types.IsConnectionConfigError(err) { var connectionConfigError *irodsclient_types.ConnectionConfigError if errors.As(err, &connectionConfigError) { - commons.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server (host: %q, port: %d)!\n", connectionConfigError.Config.Host, connectionConfigError.Config.Port) + commons.PrintErrorf("Failed to establish a connection to iRODS server (host: %q, port: %d)!\n", connectionConfigError.Config.Host, connectionConfigError.Config.Port) } else { - commons.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server!\n") + commons.PrintErrorf("Failed to establish a connection to iRODS server!\n") } } else if irodsclient_types.IsConnectionError(err) { - commons.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server!\n") + commons.PrintErrorf("Failed to establish a connection to iRODS server!\n") } else if irodsclient_types.IsConnectionPoolFullError(err) { var connectionPoolFullError *irodsclient_types.ConnectionPoolFullError if errors.As(err, &connectionPoolFullError) { - commons.Fprintf(os.Stderr, "Failed to establish a new connection to iRODS server as connection pool is full (occupied: %d, max: %d)!\n", connectionPoolFullError.Occupied, connectionPoolFullError.Max) + commons.PrintErrorf("Failed to establish a new connection to iRODS server as connection pool is full (occupied: %d, max: %d)!\n", connectionPoolFullError.Occupied, connectionPoolFullError.Max) } else { - commons.Fprintf(os.Stderr, "Failed to establish a new connection to iRODS server as connection pool is full!\n") + commons.PrintErrorf("Failed to establish a new connection to iRODS server as connection pool is full!\n") } } else if irodsclient_types.IsAuthError(err) { var authError *irodsclient_types.AuthError if errors.As(err, &authError) { - commons.Fprintf(os.Stderr, "Authentication failed (auth scheme: %q, username: %q, zone: %q)!\n", authError.Config.AuthenticationScheme, authError.Config.ClientUser, authError.Config.ClientZone) + commons.PrintErrorf("Authentication failed (auth scheme: %q, username: %q, zone: %q)!\n", authError.Config.AuthenticationScheme, authError.Config.ClientUser, authError.Config.ClientZone) } else { - commons.Fprintf(os.Stderr, "Authentication failed!\n") + commons.PrintErrorf("Authentication failed!\n") } } else if irodsclient_types.IsFileNotFoundError(err) { var fileNotFoundError *irodsclient_types.FileNotFoundError if errors.As(err, &fileNotFoundError) { - commons.Fprintf(os.Stderr, "File or directory %q is not found!\n", fileNotFoundError.Path) + commons.PrintErrorf("File or directory %q is not found!\n", fileNotFoundError.Path) } else { - commons.Fprintf(os.Stderr, "File or directory is not found!\n") + commons.PrintErrorf("File or directory is not found!\n") } } else if irodsclient_types.IsCollectionNotEmptyError(err) { var collectionNotEmptyError *irodsclient_types.CollectionNotEmptyError if errors.As(err, &collectionNotEmptyError) { - commons.Fprintf(os.Stderr, "Directory %q is not empty!\n", collectionNotEmptyError.Path) + commons.PrintErrorf("Directory %q is not empty!\n", collectionNotEmptyError.Path) } else { - commons.Fprintf(os.Stderr, "Directory is not empty!\n") + commons.PrintErrorf("Directory is not empty!\n") } } else if irodsclient_types.IsFileAlreadyExistError(err) { var fileAlreadyExistError *irodsclient_types.FileAlreadyExistError if errors.As(err, &fileAlreadyExistError) { - commons.Fprintf(os.Stderr, "File or directory %q already exists!\n", fileAlreadyExistError.Path) + commons.PrintErrorf("File or directory %q already exists!\n", fileAlreadyExistError.Path) } else { - commons.Fprintf(os.Stderr, "File or directory already exists!\n") + commons.PrintErrorf("File or directory already exists!\n") } } else if irodsclient_types.IsTicketNotFoundError(err) { var ticketNotFoundError *irodsclient_types.TicketNotFoundError if errors.As(err, &ticketNotFoundError) { - commons.Fprintf(os.Stderr, "Ticket %q is not found!\n", ticketNotFoundError.Ticket) + commons.PrintErrorf("Ticket %q is not found!\n", ticketNotFoundError.Ticket) } else { - commons.Fprintf(os.Stderr, "Ticket is not found!\n") + commons.PrintErrorf("Ticket is not found!\n") } } else if irodsclient_types.IsUserNotFoundError(err) { var userNotFoundError *irodsclient_types.UserNotFoundError if errors.As(err, &userNotFoundError) { - commons.Fprintf(os.Stderr, "User %q is not found!\n", userNotFoundError.Name) + commons.PrintErrorf("User %q is not found!\n", userNotFoundError.Name) } else { - commons.Fprintf(os.Stderr, "User is not found!\n") + commons.PrintErrorf("User is not found!\n") } } else if irodsclient_types.IsIRODSError(err) { var irodsError *irodsclient_types.IRODSError if errors.As(err, &irodsError) { - commons.Fprintf(os.Stderr, "iRODS Error (code: '%d', message: %q)\n", irodsError.Code, irodsError.Error()) + commons.PrintErrorf("iRODS Error (code: '%d', message: %q)\n", irodsError.Code, irodsError.Error()) } else { - commons.Fprintf(os.Stderr, "iRODS Error!\n") + commons.PrintErrorf("iRODS Error!\n") } } else if commons.IsNotDirError(err) { var notDirError *commons.NotDirError if errors.As(err, ¬DirError) { - commons.Fprintf(os.Stderr, "Destination %q is not a directory!\n", notDirError.Path) + commons.PrintErrorf("Destination %q is not a directory!\n", notDirError.Path) } else { - commons.Fprintf(os.Stderr, "Destination is not a directory!\n") + commons.PrintErrorf("Destination is not a directory!\n") } } else { - commons.Fprintf(os.Stderr, "Unexpected error!\nError Trace:\n - %+v\n", err) + commons.PrintErrorf("Unexpected error!\nError Trace:\n - %+v\n", err) } - //commons.Fprintf(os.Stderr, "\nError Trace:\n - %+v\n", err) os.Exit(1) } } diff --git a/commons/print.go b/commons/print.go index 0757232..a9bc9b4 100644 --- a/commons/print.go +++ b/commons/print.go @@ -65,9 +65,23 @@ func Println(a ...any) (n int, err error) { return fmt.Fprintln(terminalOutput, a...) } -func Fprintf(w io.Writer, format string, a ...any) (n int, err error) { +func Fprintf(w io.Writer, format string, a ...any) (int, error) { terminalOutput.Lock() defer terminalOutput.Unlock() return fmt.Fprintf(w, format, a...) } + +func PrintErrorf(format string, a ...any) (int, error) { + terminalOutput.Lock() + defer terminalOutput.Unlock() + + red := "\033[31m" + reset := "\033[0m" + + fmt.Fprint(os.Stderr, red) + n, err := fmt.Fprintf(os.Stderr, format, a...) + fmt.Fprint(os.Stderr, reset) + + return n, err +} diff --git a/commons/retry.go b/commons/retry.go index 7c548d7..1a28969 100644 --- a/commons/retry.go +++ b/commons/retry.go @@ -26,10 +26,10 @@ func RunWithRetry(retry int, retryInterval int) error { } logger.Errorf("%+v", err) - Fprintf(os.Stderr, "Error: %+v\n", err) + PrintErrorf("%+v\n", err) logger.Errorf("Waiting %d seconds for next try...", retryInterval) - Fprintf(os.Stderr, "Waiting %d seconds for next try...", retryInterval) + PrintErrorf("Waiting %d seconds for next try...", retryInterval) sleepTime := time.Duration(retryInterval * int(time.Second)) time.Sleep(sleepTime) From 7f8ff4c7c35567b609e7249c99232e7d7311c8e4 Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Thu, 22 Aug 2024 15:14:58 -0700 Subject: [PATCH 14/16] Fix typo --- cmd/subcmd/env.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/subcmd/env.go b/cmd/subcmd/env.go index fcfa0fd..cf66623 100644 --- a/cmd/subcmd/env.go +++ b/cmd/subcmd/env.go @@ -27,12 +27,12 @@ func AddEnvCommand(rootCmd *cobra.Command) { } func processEnvCommand(command *cobra.Command, args []string) error { - bun, err := NewBunCommand(command, args) + env, err := NewEnvCommand(command, args) if err != nil { return err } - return bun.Process() + return env.Process() } type EnvCommand struct { From 52ade13fd26858dab9fb706ff9bda49750fbdfba Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Thu, 22 Aug 2024 16:23:06 -0700 Subject: [PATCH 15/16] Fix copy-sftp-id bug --- cmd/subcmd/copy-sftp-id.go | 6 +++--- go.mod | 2 +- go.sum | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cmd/subcmd/copy-sftp-id.go b/cmd/subcmd/copy-sftp-id.go index d572a17..16eac31 100644 --- a/cmd/subcmd/copy-sftp-id.go +++ b/cmd/subcmd/copy-sftp-id.go @@ -102,7 +102,7 @@ func (copy *CopySftpIdCommand) Process() error { err = copy.copySftpId(identityFiles) if err != nil { - return xerrors.Errorf("failed to copy sftp-ID: %w", err) + return xerrors.Errorf("failed to copy sftp public key: %w", err) } return nil @@ -174,7 +174,7 @@ func (copy *CopySftpIdCommand) readAuthorizedKeys(authorizedKeyPath string) ([]s contentBuffer := bytes.Buffer{} - _, err := copy.filesystem.DownloadFileToBuffer(authorizedKeyPath, "", contentBuffer, true, nil) + _, err := copy.filesystem.DownloadFileToBuffer(authorizedKeyPath, "", &contentBuffer, true, nil) if err != nil { return nil, xerrors.Errorf("failed to read file %q: %w", authorizedKeyPath, err) } @@ -310,7 +310,7 @@ func (copy *CopySftpIdCommand) copySftpId(identityFiles []string) error { logger.Debugf("writing authorized_keys %q on iRODS for user %q", authorizedKeyPath, copy.account.ClientUser) - _, err := copy.filesystem.UploadFileFromBuffer(contentBuf, authorizedKeyPath, "", false, true, true, nil) + _, err := copy.filesystem.UploadFileFromBuffer(&contentBuf, authorizedKeyPath, "", false, true, true, nil) if err != nil { return xerrors.Errorf("failed to update keys in %q: %w", authorizedKeyPath, err) } diff --git a/go.mod b/go.mod index 4823202..eb4191d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/creativeprojects/go-selfupdate v1.0.1 - github.com/cyverse/go-irodsclient v0.14.13 + github.com/cyverse/go-irodsclient v0.14.14 github.com/dustin/go-humanize v1.0.1 github.com/gliderlabs/ssh v0.3.5 github.com/jedib0t/go-pretty/v6 v6.3.1 diff --git a/go.sum b/go.sum index b31cc64..35f6b15 100644 --- a/go.sum +++ b/go.sum @@ -9,10 +9,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creativeprojects/go-selfupdate v1.0.1 h1:5Un4MTv4puCR5GBgkDLC14J72fljGuMC60E63cFGq1o= github.com/creativeprojects/go-selfupdate v1.0.1/go.mod h1:nm7AWUJfrfYt/SB97NAcMhR0KEpPqlrVHXkWFti+ezw= -github.com/cyverse/go-irodsclient v0.14.12 h1:CTrS/pl9ADAuF5De1s4eB2Y8et29SSae7KYW09M8PZc= -github.com/cyverse/go-irodsclient v0.14.12/go.mod h1:eBXha3cwfrM0p1ijYVqsrLJQHpRwTfpA4c5dKCQsQFc= -github.com/cyverse/go-irodsclient v0.14.13 h1:PIXwZTajiOXUwmFvZLIVAFQpWsV0CXiixDDYVvdeEYU= -github.com/cyverse/go-irodsclient v0.14.13/go.mod h1:eBXha3cwfrM0p1ijYVqsrLJQHpRwTfpA4c5dKCQsQFc= +github.com/cyverse/go-irodsclient v0.14.14 h1:Q9Bl8lu3SIUjGlAIPyGmj71QBQM8KmXwnYmFFa7xsl4= +github.com/cyverse/go-irodsclient v0.14.14/go.mod h1:eBXha3cwfrM0p1ijYVqsrLJQHpRwTfpA4c5dKCQsQFc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 0d05645d348f3387a2aa4b4369b37e1e39ce4609 Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Thu, 22 Aug 2024 16:30:34 -0700 Subject: [PATCH 16/16] bump up version to v0.9.11 --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 26e2f69..9781ff2 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -v0.9.10 +v0.9.11