From ee6acb8932aa47957b6eef86eda09c3bc84a7032 Mon Sep 17 00:00:00 2001 From: Illyoung Choi Date: Fri, 20 Sep 2024 14:20:09 -0700 Subject: [PATCH] Refactor config --- cmd/flag/common.go | 168 ++++++++--- cmd/subcmd/addmeta.go | 4 +- cmd/subcmd/bclean.go | 4 +- cmd/subcmd/bput.go | 61 ++-- cmd/subcmd/bun.go | 4 +- cmd/subcmd/cat.go | 19 +- cmd/subcmd/cd.go | 4 +- cmd/subcmd/copy-sftp-id.go | 2 +- cmd/subcmd/cp.go | 99 +++--- cmd/subcmd/env.go | 84 ++++-- cmd/subcmd/get.go | 91 +++--- cmd/subcmd/init.go | 51 +++- cmd/subcmd/ls.go | 21 +- cmd/subcmd/lsmeta.go | 4 +- cmd/subcmd/lsticket.go | 2 +- cmd/subcmd/mkdir.go | 4 +- cmd/subcmd/mkticket.go | 4 +- cmd/subcmd/modticket.go | 2 +- cmd/subcmd/mv.go | 6 +- cmd/subcmd/passwd.go | 2 +- cmd/subcmd/ps.go | 2 +- cmd/subcmd/put.go | 80 +++-- cmd/subcmd/rm.go | 4 +- cmd/subcmd/rmdir.go | 4 +- cmd/subcmd/rmmeta.go | 6 +- cmd/subcmd/rmticket.go | 2 +- cmd/subcmd/svrinfo.go | 2 +- cmd/subcmd/touch.go | 4 +- commons/bundle_transfer.go | 6 +- commons/commands.go | 595 +++---------------------------------- commons/config.go | 23 +- commons/retry.go | 4 +- commons/staging.go | 6 +- commons/transfer_report.go | 53 ++-- go.mod | 3 +- go.sum | 5 +- 36 files changed, 537 insertions(+), 898 deletions(-) diff --git a/cmd/flag/common.go b/cmd/flag/common.go index 66e601a..f94d235 100644 --- a/cmd/flag/common.go +++ b/cmd/flag/common.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + irodsclient_config "github.com/cyverse/go-irodsclient/config" "github.com/cyverse/gocommands/commons" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -73,6 +74,31 @@ func GetCommonFlagValues(command *cobra.Command) *CommonFlagValues { return &commonFlagValues } +func getLogrusLogLevel(irodsLogLevel int) log.Level { + switch irodsLogLevel { + case 0: + return log.PanicLevel + case 1: + return log.FatalLevel + case 2, 3: + return log.ErrorLevel + case 4, 5, 6: + return log.WarnLevel + case 7: + return log.InfoLevel + case 8: + return log.DebugLevel + case 9, 10: + return log.TraceLevel + } + + if irodsLogLevel < 0 { + return log.PanicLevel + } + + return log.TraceLevel +} + func setLogLevel(command *cobra.Command) { myCommonFlagValues := GetCommonFlagValues(command) @@ -108,59 +134,122 @@ func ProcessCommonFlags(command *cobra.Command) (bool, error) { return false, nil // stop here } + // init config + err := commons.InitEnvironmentManager() + if err != nil { + return false, xerrors.Errorf("failed to init environment manager: %w", err) + } + + environmentManager := commons.GetEnvironmentManager() + logger.Debugf("use sessionID - %d", myCommonFlagValues.SessionID) - commons.SetSessionID(myCommonFlagValues.SessionID) + environmentManager.SetPPID(myCommonFlagValues.SessionID) - readConfig := false + configFilePath := "" + + // find config file location from env + if irodsEnvironmentFileEnvVal, ok := os.LookupEnv(IRODSEnvironmentFileEnvKey); ok { + if len(irodsEnvironmentFileEnvVal) > 0 { + configFilePath = irodsEnvironmentFileEnvVal + } + } + + // user defined config file if len(myCommonFlagValues.ConfigFilePath) > 0 { - // user defined config file - err := commons.LoadConfigFromFile(myCommonFlagValues.ConfigFilePath) + configFilePath = myCommonFlagValues.ConfigFilePath + } + + // load config + if len(configFilePath) > 0 { + configFilePath, err = commons.ExpandHomeDir(configFilePath) if err != nil { - return false, xerrors.Errorf("failed to load config from file %q: %w", myCommonFlagValues.ConfigFilePath, err) // stop here + return false, xerrors.Errorf("failed to expand home directory for %q: %w", configFilePath, err) } - readConfig = true - } else { - // read config path from env - // then read config - if irodsEnvironmentFileEnvVal, ok := os.LookupEnv(IRODSEnvironmentFileEnvKey); ok { - if len(irodsEnvironmentFileEnvVal) > 0 { - err := commons.LoadConfigFromFile(irodsEnvironmentFileEnvVal) - if err != nil { - return false, xerrors.Errorf("failed to load config file %q: %w", irodsEnvironmentFileEnvVal, err) // stop here - } - - readConfig = true + status, err := os.Stat(configFilePath) + if err != nil { + if os.IsNotExist(err) { + return false, xerrors.Errorf("config path %q does not exist", configFilePath) } + + return false, xerrors.Errorf("failed to stat %q: %w", configFilePath, err) } - // read config from default icommands config path - if !readConfig { - // auto detect - err := commons.LoadConfigFromFile("~/.irods") + if status.IsDir() { + // config root + err = environmentManager.SetEnvironmentDirPath(configFilePath) + if err != nil { + return false, xerrors.Errorf("failed to set configuration root directory %q: %w", configFilePath, err) + } + + // load + err = environmentManager.Load() if err != nil { - logger.Debug(err) - // ignore error + return false, xerrors.Errorf("failed to load configuration file %q: %w", environmentManager.EnvironmentFilePath, err) + } + } else { + // config file + if commons.IsYAMLFile(configFilePath) { + // yaml + yamlConfig, err := irodsclient_config.NewConfigFromYAMLFile(configFilePath) + if err != nil { + return false, xerrors.Errorf("failed to load YAML config from file %q: %w", configFilePath, err) + } + + // load + err = environmentManager.Load() + if err != nil { + return false, xerrors.Errorf("failed to load configuration file %q: %w", environmentManager.EnvironmentFilePath, err) + } + + // overwrite + environmentManager.Environment = yamlConfig } else { - readConfig = true + // json + jsonConfig, err := irodsclient_config.NewConfigFromJSONFile(configFilePath) + if err != nil { + return false, xerrors.Errorf("failed to load JSON config from file %q: %w", configFilePath, err) + } + + environmentManager.Environment = jsonConfig + environmentManager.EnvironmentFilePath = configFilePath + + // load + err = environmentManager.Load() + if err != nil { + return false, xerrors.Errorf("failed to load configuration file %q: %w", environmentManager.EnvironmentFilePath, err) + } } } + } else { + // default + // load + err = environmentManager.Load() + if err != nil { + return false, xerrors.Errorf("failed to load configuration file %q: %w", environmentManager.EnvironmentFilePath, err) + } } - // set default config - if !readConfig { - commons.SetDefaultConfigIfEmpty() + // load config from env + envConfig, err := irodsclient_config.OverwriteConfigFromEnv(environmentManager.Environment) + if err != nil { + return false, xerrors.Errorf("failed to load config from environment: %w", err) } - // re-configure level - setLogLevel(command) + // overwrite + environmentManager.Environment = envConfig - err := commons.LoadAndOverwriteConfigFromEnv() + sessionConfig, err := environmentManager.GetSessionConfig() if err != nil { - return false, xerrors.Errorf("failed to load config from environment: %w", err) // stop here + return false, xerrors.Errorf("failed to get session config: %w", err) + } + + if sessionConfig.LogLevel > 0 { + // set log level + log.SetLevel(getLogrusLogLevel(sessionConfig.LogLevel)) } - // re-configure level + // prioritize log level user set via command-line argument setLogLevel(command) if retryFlagValues.RetryChild { @@ -171,20 +260,9 @@ func ProcessCommonFlags(command *cobra.Command) (bool, error) { } } - appConfig := commons.GetConfig() - - syncAccount := false if myCommonFlagValues.ResourceUpdated { - appConfig.DefaultResource = myCommonFlagValues.Resource - logger.Debugf("use default resource server %q", appConfig.DefaultResource) - syncAccount = true - } - - if syncAccount { - err := commons.SyncAccount() - if err != nil { - return false, err - } + environmentManager.Environment.DefaultResource = myCommonFlagValues.Resource + logger.Debugf("use default resource server %q", myCommonFlagValues.Resource) } return true, nil // contiue diff --git a/cmd/subcmd/addmeta.go b/cmd/subcmd/addmeta.go index c9729be..a8e70b9 100644 --- a/cmd/subcmd/addmeta.go +++ b/cmd/subcmd/addmeta.go @@ -85,7 +85,7 @@ func (addMeta *AddMetaCommand) Process() error { } // Create a file system - addMeta.account = commons.GetAccount() + addMeta.account = commons.GetSessionConfig().ToIRODSAccount() addMeta.filesystem, err = commons.GetIRODSFSClient(addMeta.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -125,7 +125,7 @@ func (addMeta *AddMetaCommand) addMetaToPath(targetPath string, attribute string cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := addMeta.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) logger.Debugf("add metadata to path %q (attr %q, value %q, unit %q)", targetPath, attribute, value, unit) diff --git a/cmd/subcmd/bclean.go b/cmd/subcmd/bclean.go index 6a54765..872321e 100644 --- a/cmd/subcmd/bclean.go +++ b/cmd/subcmd/bclean.go @@ -86,7 +86,7 @@ func (bclean *BcleanCommand) Process() error { } // Create a file system - bclean.account = commons.GetAccount() + bclean.account = commons.GetSessionConfig().ToIRODSAccount() bclean.filesystem, err = commons.GetIRODSFSClient(bclean.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -124,7 +124,7 @@ func (bclean *BcleanCommand) cleanOne(targetPath string) { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := bclean.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) if commons.IsStagingDirInTargetPath(targetPath) { diff --git a/cmd/subcmd/bput.go b/cmd/subcmd/bput.go index 66056b8..703947c 100644 --- a/cmd/subcmd/bput.go +++ b/cmd/subcmd/bput.go @@ -163,7 +163,7 @@ func (bput *BputCommand) Process() error { } // Create a file system - bput.account = commons.GetAccount() + bput.account = commons.GetSessionConfig().ToIRODSAccount() bput.filesystem, err = commons.GetIRODSFSClientAdvanced(bput.account, bput.maxConnectionNum, bput.parallelTransferFlagValues.TCPBufferSize) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -212,7 +212,7 @@ func (bput *BputCommand) Process() error { } // bundle transfer manager - bput.bundleTransferManager = commons.NewBundleTransferManager(bput.filesystem, bput.transferReportManager, bput.targetPath, localBundleRootPath, bput.bundleTransferFlagValues.MinFileNum, bput.bundleTransferFlagValues.MaxFileNum, bput.bundleTransferFlagValues.MaxFileSize, bput.parallelTransferFlagValues.SingleThread, bput.parallelTransferFlagValues.ThreadNumber, bput.parallelTransferFlagValues.RedirectToResource, bput.parallelTransferFlagValues.Icat, bput.bundleTransferFlagValues.LocalTempPath, stagingDirPath, bput.bundleTransferFlagValues.NoBulkRegistration, bput.progressFlagValues.ShowProgress, bput.progressFlagValues.ShowFullPath) + bput.bundleTransferManager = commons.NewBundleTransferManager(bput.account, bput.filesystem, bput.transferReportManager, bput.targetPath, localBundleRootPath, bput.bundleTransferFlagValues.MinFileNum, bput.bundleTransferFlagValues.MaxFileNum, bput.bundleTransferFlagValues.MaxFileSize, bput.parallelTransferFlagValues.SingleThread, bput.parallelTransferFlagValues.ThreadNumber, bput.parallelTransferFlagValues.RedirectToResource, bput.parallelTransferFlagValues.Icat, bput.bundleTransferFlagValues.LocalTempPath, stagingDirPath, bput.bundleTransferFlagValues.NoBulkRegistration, bput.progressFlagValues.ShowProgress, bput.progressFlagValues.ShowFullPath) bput.bundleTransferManager.Start() // run @@ -257,7 +257,7 @@ func (bput *BputCommand) Process() error { func (bput *BputCommand) ensureTargetIsDir(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := bput.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) targetEntry, err := bput.filesystem.Stat(targetPath) @@ -286,7 +286,7 @@ func (bput *BputCommand) getStagingDir(targetPath string) (string, error) { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := bput.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) if len(bput.bundleTransferFlagValues.IRODSTempPath) > 0 { @@ -503,10 +503,10 @@ func (bput *BputCommand) putFile(sourceStat fs.FileInfo, sourcePath string) erro SourcePath: sourcePath, SourceSize: sourceStat.Size(), - DestPath: targetEntry.Path, - DestSize: targetEntry.Size, - ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), - Notes: []string{"differential", "no_hash", "same file size", "skip"}, + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"differential", "no_hash", "same file size", "skip"}, } bput.transferReportManager.AddFile(reportFile) @@ -528,17 +528,18 @@ func (bput *BputCommand) putFile(sourceStat fs.FileInfo, sourcePath string) erro // skip now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodPut, - StartAt: now, - EndAt: now, - SourcePath: sourcePath, - SourceSize: sourceStat.Size(), - SourceChecksum: hex.EncodeToString(localChecksum), - DestPath: targetEntry.Path, - DestSize: targetEntry.Size, - DestChecksum: hex.EncodeToString(targetEntry.CheckSum), - ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), - Notes: []string{"differential", "same checksum", "skip"}, + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + SourceSize: sourceStat.Size(), + SourceChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + SourceChecksum: hex.EncodeToString(localChecksum), + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"differential", "same checksum", "skip"}, } bput.transferReportManager.AddFile(reportFile) @@ -558,16 +559,16 @@ func (bput *BputCommand) putFile(sourceStat fs.FileInfo, sourcePath string) erro // skip now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodPut, - StartAt: now, - EndAt: now, - SourcePath: sourcePath, - SourceSize: sourceStat.Size(), - DestPath: targetEntry.Path, - DestSize: targetEntry.Size, - DestChecksum: hex.EncodeToString(targetEntry.CheckSum), - ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), - Notes: []string{"no_overwrite", "skip"}, + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + SourceSize: sourceStat.Size(), + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"no_overwrite", "skip"}, } bput.transferReportManager.AddFile(reportFile) @@ -726,7 +727,7 @@ func (bput *BputCommand) deleteOnSuccess(sourcePath string) error { func (bput *BputCommand) deleteExtra(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := bput.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) return bput.deleteExtraInternal(targetPath) diff --git a/cmd/subcmd/bun.go b/cmd/subcmd/bun.go index 30d3a2c..bf97070 100644 --- a/cmd/subcmd/bun.go +++ b/cmd/subcmd/bun.go @@ -89,7 +89,7 @@ func (bun *BunCommand) Process() error { } // Create a file system - bun.account = commons.GetAccount() + bun.account = commons.GetSessionConfig().ToIRODSAccount() bun.filesystem, err = commons.GetIRODSFSClient(bun.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -150,7 +150,7 @@ func (bun *BunCommand) extractOne(sourcePath string, targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := bun.account.ClientZone sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) diff --git a/cmd/subcmd/cat.go b/cmd/subcmd/cat.go index 230e4ed..268e929 100644 --- a/cmd/subcmd/cat.go +++ b/cmd/subcmd/cat.go @@ -85,24 +85,13 @@ func (cat *CatCommand) Process() error { return xerrors.Errorf("failed to input missing fields: %w", err) } - // config - appConfig := commons.GetConfig() - syncAccount := false + // Create a file system + cat.account = commons.GetSessionConfig().ToIRODSAccount() if len(cat.ticketAccessFlagValues.Name) > 0 { logger.Debugf("use ticket: %q", cat.ticketAccessFlagValues.Name) - appConfig.Ticket = cat.ticketAccessFlagValues.Name - syncAccount = true - } - - if syncAccount { - err := commons.SyncAccount() - if err != nil { - return err - } + cat.account.Ticket = cat.ticketAccessFlagValues.Name } - // Create a file system - cat.account = commons.GetAccount() cat.filesystem, err = commons.GetIRODSFSClient(cat.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -123,7 +112,7 @@ func (cat *CatCommand) Process() error { func (cat *CatCommand) catOne(sourcePath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := cat.account.ClientZone sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) sourceEntry, err := cat.filesystem.Stat(sourcePath) diff --git a/cmd/subcmd/cd.go b/cmd/subcmd/cd.go index d63f101..5261c36 100644 --- a/cmd/subcmd/cd.go +++ b/cmd/subcmd/cd.go @@ -79,7 +79,7 @@ func (cd *CdCommand) Process() error { } // Create a file system - cd.account = commons.GetAccount() + cd.account = commons.GetSessionConfig().ToIRODSAccount() cd.filesystem, err = commons.GetIRODSFSClient(cd.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -104,7 +104,7 @@ func (cd *CdCommand) changeWorkingDir(collectionPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := cd.account.ClientZone collectionPath = commons.MakeIRODSPath(cwd, home, zone, collectionPath) connection, err := cd.filesystem.GetMetadataConnection() diff --git a/cmd/subcmd/copy-sftp-id.go b/cmd/subcmd/copy-sftp-id.go index 16eac31..d1bf7a7 100644 --- a/cmd/subcmd/copy-sftp-id.go +++ b/cmd/subcmd/copy-sftp-id.go @@ -86,7 +86,7 @@ func (copy *CopySftpIdCommand) Process() error { } // Create a file system - copy.account = commons.GetAccount() + copy.account = commons.GetSessionConfig().ToIRODSAccount() copy.filesystem, err = commons.GetIRODSFSClient(copy.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) diff --git a/cmd/subcmd/cp.go b/cmd/subcmd/cp.go index 4edbf17..68606e6 100644 --- a/cmd/subcmd/cp.go +++ b/cmd/subcmd/cp.go @@ -139,7 +139,7 @@ func (cp *CpCommand) Process() error { } // Create a file system - cp.account = commons.GetAccount() + cp.account = commons.GetSessionConfig().ToIRODSAccount() cp.filesystem, err = commons.GetIRODSFSClient(cp.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -195,7 +195,7 @@ func (cp *CpCommand) Process() error { func (cp *CpCommand) ensureTargetIsDir(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := cp.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) targetEntry, err := cp.filesystem.Stat(targetPath) @@ -218,7 +218,7 @@ func (cp *CpCommand) ensureTargetIsDir(targetPath string) error { func (cp *CpCommand) copyOne(sourcePath string, targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := cp.account.ClientZone sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) @@ -267,20 +267,21 @@ func (cp *CpCommand) scheduleCopy(sourceEntry *irodsclient_fs.Entry, targetPath now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodCopy, - StartAt: now, - EndAt: now, - SourcePath: sourceEntry.Path, - SourceSize: sourceEntry.Size, - SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), - DestPath: targetPath, - - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{}, + Method: commons.TransferMethodCopy, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + + Notes: []string{}, } if targetEntry != nil { reportFile.DestSize = targetEntry.Size + reportFile.DestChecksumAlgorithm = string(targetEntry.CheckSumAlgorithm) reportFile.DestChecksum = hex.EncodeToString(targetEntry.CheckSum) } @@ -382,17 +383,19 @@ func (cp *CpCommand) copyFile(sourceEntry *irodsclient_fs.Entry, targetPath stri // skip now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodCopy, - StartAt: now, - EndAt: now, - SourcePath: sourceEntry.Path, - SourceSize: sourceEntry.Size, - SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), - DestPath: targetPath, - DestSize: targetEntry.Size, - DestChecksum: hex.EncodeToString(targetEntry.CheckSum), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"differential", "no_hash", "same file size", "skip"}, + Method: commons.TransferMethodCopy, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + DestSize: targetEntry.Size, + DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + + Notes: []string{"differential", "no_hash", "same file size", "skip"}, } cp.transferReportManager.AddFile(reportFile) @@ -407,17 +410,18 @@ func (cp *CpCommand) copyFile(sourceEntry *irodsclient_fs.Entry, targetPath stri if len(sourceEntry.CheckSum) > 0 && bytes.Equal(sourceEntry.CheckSum, targetEntry.CheckSum) { now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodCopy, - StartAt: now, - EndAt: now, - SourcePath: sourceEntry.Path, - SourceSize: sourceEntry.Size, - SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), - DestPath: targetPath, - DestSize: targetEntry.Size, - DestChecksum: hex.EncodeToString(targetEntry.CheckSum), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"differential", "same checksum", "skip"}, + Method: commons.TransferMethodCopy, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"differential", "same checksum", "skip"}, } cp.transferReportManager.AddFile(reportFile) @@ -435,17 +439,18 @@ func (cp *CpCommand) copyFile(sourceEntry *irodsclient_fs.Entry, targetPath stri if !overwrite { now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodCopy, - StartAt: now, - EndAt: now, - SourcePath: sourceEntry.Path, - SourceSize: sourceEntry.Size, - SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), - DestPath: targetPath, - DestSize: targetEntry.Size, - DestChecksum: hex.EncodeToString(targetEntry.CheckSum), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"no_overwrite", "skip"}, + Method: commons.TransferMethodCopy, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"no_overwrite", "skip"}, } cp.transferReportManager.AddFile(reportFile) @@ -578,7 +583,7 @@ func (cp *CpCommand) copyDir(sourceEntry *irodsclient_fs.Entry, targetPath strin func (cp *CpCommand) deleteExtra(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := cp.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) return cp.deleteExtraInternal(targetPath) diff --git a/cmd/subcmd/env.go b/cmd/subcmd/env.go index 2600f43..50b778c 100644 --- a/cmd/subcmd/env.go +++ b/cmd/subcmd/env.go @@ -46,6 +46,7 @@ func NewEnvCommand(command *cobra.Command, args []string) (*EnvCommand, error) { return env, nil } + func (env *EnvCommand) Process() error { cont, err := flag.ProcessCommonFlags(env.command) if err != nil { @@ -73,81 +74,106 @@ func (env *EnvCommand) printEnvironment() error { t := table.NewWriter() t.SetOutputMirror(os.Stdout) + sessionConfig, err := envMgr.GetSessionConfig() + if err != nil { + return err + } + t.AppendRows([]table.Row{ { - "iRODS Session Environment File", - envMgr.GetSessionFilePath(os.Getppid()), + "Session Environment File", + envMgr.SessionFilePath, + }, + { + "Environment File", + envMgr.EnvironmentFilePath, + }, + { + "Authentication File", + envMgr.PasswordFilePath, + }, + { + "Host", + sessionConfig.Host, + }, + { + "Port", + sessionConfig.Port, + }, + { + "Zone", + sessionConfig.ZoneName, }, { - "iRODS Environment File", - envMgr.GetEnvironmentFilePath(), + "Username", + sessionConfig.Username, }, { - "iRODS Authentication File", - envMgr.GetPasswordFilePath(), + "Client Zone", + sessionConfig.ClientZoneName, }, { - "iRODS Host", - envMgr.Environment.Host, + "Client Username", + sessionConfig.ClientUsername, }, { - "iRODS Port", - envMgr.Environment.Port, + "Default Resource", + sessionConfig.DefaultResource, }, { - "iRODS Zone", - envMgr.Environment.Zone, + "Current Working Dir", + commons.GetCWD(), }, { - "iRODS Username", - envMgr.Environment.Username, + "Home", + commons.GetHomeDir(), }, { - "iRODS Default Resource", - envMgr.Environment.DefaultResource, + "Default Hash Scheme", + sessionConfig.DefaultHashScheme, }, { - "iRODS Default Hash Scheme", - envMgr.Environment.DefaultHashScheme, + "Log Level", + sessionConfig.LogLevel, }, { - "iRODS Authentication Scheme", - envMgr.Environment.AuthenticationScheme, + "Authentication Scheme", + sessionConfig.AuthenticationScheme, }, { - "iRODS Client Server Negotiation", + "Client Server Negotiation", envMgr.Environment.ClientServerNegotiation, }, { - "iRODS Client Server Policy", + "Client Server Policy", envMgr.Environment.ClientServerPolicy, }, { - "iRODS SSL CA Certification File", + "SSL CA Certification File", envMgr.Environment.SSLCACertificateFile, }, { - "iRODS SSL CA Certification Path", + "SSL CA Certification Path", envMgr.Environment.SSLCACertificatePath, }, { - "iRODS SSL Verify Server", + "SSL Verify Server", envMgr.Environment.SSLVerifyServer, }, { - "iRODS SSL Encryption Key Size", + "SSL Encryption Key Size", envMgr.Environment.EncryptionKeySize, }, { - "iRODS SSL Encryption Key Algorithm", + "SSL Encryption Key Algorithm", envMgr.Environment.EncryptionAlgorithm, }, { - "iRODS SSL Encryption Salt Size", + "SSL Encryption Salt Size", envMgr.Environment.EncryptionSaltSize, }, { - "iRODS SSL Encryption Hash Rounds", + "SSL Encryption Hash Rounds", envMgr.Environment.EncryptionNumHashRounds, }, }, table.RowConfig{}) diff --git a/cmd/subcmd/get.go b/cmd/subcmd/get.go index 5392d6e..de08086 100644 --- a/cmd/subcmd/get.go +++ b/cmd/subcmd/get.go @@ -152,22 +152,6 @@ func (get *GetCommand) Process() error { return xerrors.Errorf("failed to input missing fields: %w", err) } - // config - appConfig := commons.GetConfig() - syncAccount := false - if len(get.ticketAccessFlagValues.Name) > 0 { - logger.Debugf("use ticket %q", get.ticketAccessFlagValues.Name) - appConfig.Ticket = get.ticketAccessFlagValues.Name - syncAccount = true - } - - if syncAccount { - err := commons.SyncAccount() - if err != nil { - return err - } - } - // handle retry if get.retryFlagValues.RetryNumber > 0 && !get.retryFlagValues.RetryChild { err := commons.RunWithRetry(get.retryFlagValues.RetryNumber, get.retryFlagValues.RetryIntervalSeconds) @@ -178,7 +162,13 @@ func (get *GetCommand) Process() error { } // Create a file system - get.account = commons.GetAccount() + get.account = commons.GetSessionConfig().ToIRODSAccount() + if len(get.ticketAccessFlagValues.Name) > 0 { + logger.Debugf("use ticket: %q", get.ticketAccessFlagValues.Name) + get.account.Ticket = get.ticketAccessFlagValues.Name + } + + get.account = commons.GetSessionConfig().ToIRODSAccount() get.filesystem, err = commons.GetIRODSFSClientAdvanced(get.account, get.maxConnectionNum, get.parallelTransferFlagValues.TCPBufferSize) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -296,7 +286,7 @@ func (get *GetCommand) deleteTransferStatusFile(targetPath string) { func (get *GetCommand) getOne(sourcePath string, targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := get.account.ClientZone sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) targetPath = commons.MakeLocalPath(targetPath) @@ -531,17 +521,17 @@ func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath strin // skip now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodGet, - StartAt: now, - EndAt: now, - SourcePath: sourceEntry.Path, - SourceSize: sourceEntry.Size, - SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), - - DestPath: targetPath, - DestSize: targetStat.Size(), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"differential", "no_hash", "same file size", "skip"}, + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + + DestPath: targetPath, + DestSize: targetStat.Size(), + Notes: []string{"differential", "no_hash", "same file size", "skip"}, } get.transferReportManager.AddFile(reportFile) @@ -563,17 +553,18 @@ func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath strin // skip now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodGet, - StartAt: now, - EndAt: now, - SourcePath: sourceEntry.Path, - SourceSize: sourceEntry.Size, - SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), - DestPath: targetPath, - DestSize: targetStat.Size(), - DestChecksum: hex.EncodeToString(localChecksum), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"differential", "same checksum", "skip"}, + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + DestSize: targetStat.Size(), + DestChecksum: hex.EncodeToString(localChecksum), + DestChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + Notes: []string{"differential", "same checksum", "skip"}, } get.transferReportManager.AddFile(reportFile) @@ -593,16 +584,16 @@ func (get *GetCommand) getFile(sourceEntry *irodsclient_fs.Entry, tempPath strin // skip now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodGet, - StartAt: now, - EndAt: now, - SourcePath: sourceEntry.Path, - SourceSize: sourceEntry.Size, - SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), - DestPath: targetPath, - DestSize: targetStat.Size(), - ChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), - Notes: []string{"no_overwrite", "skip"}, + Method: commons.TransferMethodGet, + StartAt: now, + EndAt: now, + SourcePath: sourceEntry.Path, + SourceSize: sourceEntry.Size, + SourceChecksumAlgorithm: string(sourceEntry.CheckSumAlgorithm), + SourceChecksum: hex.EncodeToString(sourceEntry.CheckSum), + DestPath: targetPath, + DestSize: targetStat.Size(), + Notes: []string{"no_overwrite", "skip"}, } get.transferReportManager.AddFile(reportFile) diff --git a/cmd/subcmd/init.go b/cmd/subcmd/init.go index 532c0c9..da42edb 100644 --- a/cmd/subcmd/init.go +++ b/cmd/subcmd/init.go @@ -1,12 +1,15 @@ package subcmd import ( + "os" + "github.com/cyverse/gocommands/cmd/flag" "github.com/cyverse/gocommands/commons" + "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" "golang.org/x/xerrors" - irodsclient_icommands "github.com/cyverse/go-irodsclient/icommands" + irodsclient_config "github.com/cyverse/go-irodsclient/config" irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" ) @@ -43,7 +46,7 @@ type InitCommand struct { initFlagValues *flag.InitFlagValues - environmentManager *irodsclient_icommands.ICommandsEnvironmentManager + environmentManager *irodsclient_config.ICommandsEnvironmentManager account *irodsclient_types.IRODSAccount } @@ -92,18 +95,21 @@ func (init *InitCommand) Process() error { if init.account.AuthenticationScheme.IsPAM() { // update pam token - init.environmentManager.PamToken = conn.GetPAMToken() + init.environmentManager.Environment.PAMToken = conn.GetPAMToken() } if updated { // save - err := commons.GetEnvironmentManager().SaveEnvironment() + manager := commons.GetEnvironmentManager() + manager.FixAuthConfiguration() + + err := manager.SaveEnvironment() if err != nil { return xerrors.Errorf("failed to save iCommands Environment: %w", err) } } else { commons.Println("gocommands is already configured for following account:") - err := commons.PrintAccount() + err := init.PrintAccount() if err != nil { return xerrors.Errorf("failed to print account info: %w", err) } @@ -111,3 +117,38 @@ func (init *InitCommand) Process() error { return nil } + +func (init *InitCommand) PrintAccount() error { + envMgr := commons.GetEnvironmentManager() + if envMgr == nil { + return xerrors.Errorf("environment is not set") + } + + t := table.NewWriter() + t.SetOutputMirror(os.Stdout) + + t.AppendRows([]table.Row{ + { + "iRODS Host", + envMgr.Environment.Host, + }, + { + "iRODS Port", + envMgr.Environment.Port, + }, + { + "iRODS Zone", + envMgr.Environment.ZoneName, + }, + { + "iRODS Username", + envMgr.Environment.Username, + }, + { + "iRODS Authentication Scheme", + envMgr.Environment.AuthenticationScheme, + }, + }, table.RowConfig{}) + t.Render() + return nil +} diff --git a/cmd/subcmd/ls.go b/cmd/subcmd/ls.go index 95b36c2..afc3b81 100644 --- a/cmd/subcmd/ls.go +++ b/cmd/subcmd/ls.go @@ -111,24 +111,13 @@ func (ls *LsCommand) Process() error { return xerrors.Errorf("failed to input missing fields: %w", err) } - // config - appConfig := commons.GetConfig() - syncAccount := false + // Create a file system + ls.account = commons.GetSessionConfig().ToIRODSAccount() if len(ls.ticketAccessFlagValues.Name) > 0 { - logger.Debugf("use ticket %q", ls.ticketAccessFlagValues.Name) - appConfig.Ticket = ls.ticketAccessFlagValues.Name - syncAccount = true - } - - if syncAccount { - err := commons.SyncAccount() - if err != nil { - return err - } + logger.Debugf("use ticket: %q", ls.ticketAccessFlagValues.Name) + ls.account.Ticket = ls.ticketAccessFlagValues.Name } - // Create a file system - ls.account = commons.GetAccount() ls.filesystem, err = commons.GetIRODSFSClient(ls.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -167,7 +156,7 @@ func (ls *LsCommand) requireDecryption(sourcePath string) bool { func (ls *LsCommand) listOne(sourcePath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := ls.account.ClientZone sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) sourceEntry, err := ls.filesystem.Stat(sourcePath) diff --git a/cmd/subcmd/lsmeta.go b/cmd/subcmd/lsmeta.go index ba414b5..c324074 100644 --- a/cmd/subcmd/lsmeta.go +++ b/cmd/subcmd/lsmeta.go @@ -79,7 +79,7 @@ func (lsMeta *LsMetaCommand) Process() error { } // Create a file system - lsMeta.account = commons.GetAccount() + lsMeta.account = commons.GetSessionConfig().ToIRODSAccount() lsMeta.filesystem, err = commons.GetIRODSFSClient(lsMeta.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -101,7 +101,7 @@ func (lsMeta *LsMetaCommand) Process() error { func (lsMeta *LsMetaCommand) listMetaForPath(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := lsMeta.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) metas, err := lsMeta.filesystem.ListMetadata(targetPath) diff --git a/cmd/subcmd/lsticket.go b/cmd/subcmd/lsticket.go index 55d46ee..6ce6f03 100644 --- a/cmd/subcmd/lsticket.go +++ b/cmd/subcmd/lsticket.go @@ -81,7 +81,7 @@ func (lsTicket *LsTicketCommand) Process() error { } // Create a file system - lsTicket.account = commons.GetAccount() + lsTicket.account = commons.GetSessionConfig().ToIRODSAccount() lsTicket.filesystem, err = commons.GetIRODSFSClient(lsTicket.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) diff --git a/cmd/subcmd/mkdir.go b/cmd/subcmd/mkdir.go index b07e1f9..b8a02c9 100644 --- a/cmd/subcmd/mkdir.go +++ b/cmd/subcmd/mkdir.go @@ -79,7 +79,7 @@ func (mkDir *MkDirCommand) Process() error { } // Create a file system - mkDir.account = commons.GetAccount() + mkDir.account = commons.GetSessionConfig().ToIRODSAccount() mkDir.filesystem, err = commons.GetIRODSFSClient(mkDir.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -105,7 +105,7 @@ func (mkDir *MkDirCommand) makeOne(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := mkDir.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) connection, err := mkDir.filesystem.GetMetadataConnection() diff --git a/cmd/subcmd/mkticket.go b/cmd/subcmd/mkticket.go index 2b850b6..42e9fae 100644 --- a/cmd/subcmd/mkticket.go +++ b/cmd/subcmd/mkticket.go @@ -79,7 +79,7 @@ func (mkTicket *MkTicketCommand) Process() error { } // Create a file system - mkTicket.account = commons.GetAccount() + mkTicket.account = commons.GetSessionConfig().ToIRODSAccount() mkTicket.filesystem, err = commons.GetIRODSFSClient(mkTicket.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -105,7 +105,7 @@ func (mkTicket *MkTicketCommand) makeTicket(ticketName string, ticketType types. cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := mkTicket.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) err := mkTicket.filesystem.CreateTicket(ticketName, ticketType, targetPath) diff --git a/cmd/subcmd/modticket.go b/cmd/subcmd/modticket.go index ba64abe..ee7bf48 100644 --- a/cmd/subcmd/modticket.go +++ b/cmd/subcmd/modticket.go @@ -78,7 +78,7 @@ func (modTicket *ModTicketCommand) Process() error { } // Create a file system - modTicket.account = commons.GetAccount() + modTicket.account = commons.GetSessionConfig().ToIRODSAccount() modTicket.filesystem, err = commons.GetIRODSFSClient(modTicket.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) diff --git a/cmd/subcmd/mv.go b/cmd/subcmd/mv.go index b882af7..cecad55 100644 --- a/cmd/subcmd/mv.go +++ b/cmd/subcmd/mv.go @@ -76,7 +76,7 @@ func (mv *MvCommand) Process() error { } // Create a file system - mv.account = commons.GetAccount() + mv.account = commons.GetSessionConfig().ToIRODSAccount() mv.filesystem, err = commons.GetIRODSFSClient(mv.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -105,7 +105,7 @@ func (mv *MvCommand) Process() error { func (mv *MvCommand) ensureTargetIsDir(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := mv.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) targetEntry, err := mv.filesystem.Stat(targetPath) @@ -128,7 +128,7 @@ func (mv *MvCommand) ensureTargetIsDir(targetPath string) error { func (mv *MvCommand) moveOne(sourcePath string, targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := mv.account.ClientZone sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath) targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) diff --git a/cmd/subcmd/passwd.go b/cmd/subcmd/passwd.go index 70be7af..36fa5c8 100644 --- a/cmd/subcmd/passwd.go +++ b/cmd/subcmd/passwd.go @@ -68,7 +68,7 @@ func (passwd *PasswdCommand) Process() error { } // Create a connection - passwd.account = commons.GetAccount() + passwd.account = commons.GetSessionConfig().ToIRODSAccount() passwd.filesystem, err = commons.GetIRODSFSClient(passwd.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) diff --git a/cmd/subcmd/ps.go b/cmd/subcmd/ps.go index 439a797..1ec4d21 100644 --- a/cmd/subcmd/ps.go +++ b/cmd/subcmd/ps.go @@ -78,7 +78,7 @@ func (ps *PsCommand) Process() error { } // Create a connection - ps.account = commons.GetAccount() + ps.account = commons.GetSessionConfig().ToIRODSAccount() ps.filesystem, err = commons.GetIRODSFSClient(ps.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) diff --git a/cmd/subcmd/put.go b/cmd/subcmd/put.go index b00fcf4..7c91445 100644 --- a/cmd/subcmd/put.go +++ b/cmd/subcmd/put.go @@ -153,22 +153,6 @@ func (put *PutCommand) Process() error { return xerrors.Errorf("failed to input missing fields: %w", err) } - // config - appConfig := commons.GetConfig() - syncAccount := false - if len(put.ticketAccessFlagValues.Name) > 0 { - logger.Debugf("use ticket %q", put.ticketAccessFlagValues.Name) - appConfig.Ticket = put.ticketAccessFlagValues.Name - syncAccount = true - } - - if syncAccount { - err := commons.SyncAccount() - if err != nil { - return err - } - } - // handle retry if put.retryFlagValues.RetryNumber > 0 && !put.retryFlagValues.RetryChild { err = commons.RunWithRetry(put.retryFlagValues.RetryNumber, put.retryFlagValues.RetryIntervalSeconds) @@ -179,7 +163,12 @@ func (put *PutCommand) Process() error { } // Create a file system - put.account = commons.GetAccount() + put.account = commons.GetSessionConfig().ToIRODSAccount() + if len(put.ticketAccessFlagValues.Name) > 0 { + logger.Debugf("use ticket: %q", put.ticketAccessFlagValues.Name) + put.account.Ticket = put.ticketAccessFlagValues.Name + } + put.filesystem, err = commons.GetIRODSFSClientAdvanced(put.account, put.maxConnectionNum, put.parallelTransferFlagValues.TCPBufferSize) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -252,7 +241,7 @@ func (put *PutCommand) Process() error { func (put *PutCommand) ensureTargetIsDir(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := put.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) targetEntry, err := put.filesystem.Stat(targetPath) @@ -317,7 +306,7 @@ func (put *PutCommand) requireEncryption(targetPath string, parentEncryption boo func (put *PutCommand) putOne(sourcePath string, targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := put.account.ClientZone sourcePath = commons.MakeLocalPath(sourcePath) targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) @@ -537,10 +526,10 @@ func (put *PutCommand) putFile(sourceStat fs.FileInfo, sourcePath string, tempPa SourcePath: sourcePath, SourceSize: sourceStat.Size(), - DestPath: targetEntry.Path, - DestSize: targetEntry.Size, - ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), - Notes: []string{"differential", "no_hash", "same file size", "skip"}, + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"differential", "no_hash", "same file size", "skip"}, } put.transferReportManager.AddFile(reportFile) @@ -562,17 +551,18 @@ func (put *PutCommand) putFile(sourceStat fs.FileInfo, sourcePath string, tempPa // skip now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodPut, - StartAt: now, - EndAt: now, - SourcePath: sourcePath, - SourceSize: sourceStat.Size(), - SourceChecksum: hex.EncodeToString(localChecksum), - DestPath: targetEntry.Path, - DestSize: targetEntry.Size, - DestChecksum: hex.EncodeToString(targetEntry.CheckSum), - ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), - Notes: []string{"differential", "same checksum", "skip"}, + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + SourceSize: sourceStat.Size(), + SourceChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + SourceChecksum: hex.EncodeToString(localChecksum), + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"differential", "same checksum", "skip"}, } put.transferReportManager.AddFile(reportFile) @@ -592,16 +582,16 @@ func (put *PutCommand) putFile(sourceStat fs.FileInfo, sourcePath string, tempPa // skip now := time.Now() reportFile := &commons.TransferReportFile{ - Method: commons.TransferMethodPut, - StartAt: now, - EndAt: now, - SourcePath: sourcePath, - SourceSize: sourceStat.Size(), - DestPath: targetEntry.Path, - DestSize: targetEntry.Size, - DestChecksum: hex.EncodeToString(targetEntry.CheckSum), - ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), - Notes: []string{"no_overwrite", "skip"}, + Method: commons.TransferMethodPut, + StartAt: now, + EndAt: now, + SourcePath: sourcePath, + SourceSize: sourceStat.Size(), + DestPath: targetEntry.Path, + DestSize: targetEntry.Size, + DestChecksum: hex.EncodeToString(targetEntry.CheckSum), + DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm), + Notes: []string{"no_overwrite", "skip"}, } put.transferReportManager.AddFile(reportFile) @@ -790,7 +780,7 @@ func (put *PutCommand) deleteOnSuccess(sourcePath string) error { func (put *PutCommand) deleteExtra(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := put.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) return put.deleteExtraInternal(targetPath) diff --git a/cmd/subcmd/rm.go b/cmd/subcmd/rm.go index e80890b..83e1231 100644 --- a/cmd/subcmd/rm.go +++ b/cmd/subcmd/rm.go @@ -81,7 +81,7 @@ func (rm *RmCommand) Process() error { } // Create a file system - rm.account = commons.GetAccount() + rm.account = commons.GetSessionConfig().ToIRODSAccount() rm.filesystem, err = commons.GetIRODSFSClient(rm.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -107,7 +107,7 @@ func (rm *RmCommand) removeOne(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := rm.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) targetEntry, err := rm.filesystem.Stat(targetPath) diff --git a/cmd/subcmd/rmdir.go b/cmd/subcmd/rmdir.go index 54943c6..ec891fc 100644 --- a/cmd/subcmd/rmdir.go +++ b/cmd/subcmd/rmdir.go @@ -81,7 +81,7 @@ func (rmDir *RmDirCommand) Process() error { } // Create a file system - rmDir.account = commons.GetAccount() + rmDir.account = commons.GetSessionConfig().ToIRODSAccount() rmDir.filesystem, err = commons.GetIRODSFSClient(rmDir.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -108,7 +108,7 @@ func (rmDir *RmDirCommand) removeOne(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := rmDir.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) targetEntry, err := rmDir.filesystem.Stat(targetPath) diff --git a/cmd/subcmd/rmmeta.go b/cmd/subcmd/rmmeta.go index f5983db..3e41fa4 100644 --- a/cmd/subcmd/rmmeta.go +++ b/cmd/subcmd/rmmeta.go @@ -80,7 +80,7 @@ func (rmMeta *RmMetaCommand) Process() error { } // Create a file system - rmMeta.account = commons.GetAccount() + rmMeta.account = commons.GetSessionConfig().ToIRODSAccount() rmMeta.filesystem, err = commons.GetIRODSFSClient(rmMeta.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -187,7 +187,7 @@ func (rmMeta *RmMetaCommand) removeMetaFromPath(targetPath string, avuid int64) cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := rmMeta.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) err := rmMeta.filesystem.DeleteMetadata(targetPath, avuid) @@ -243,7 +243,7 @@ func (rmMeta *RmMetaCommand) removeMetaFromPathByName(targetPath string, attr st cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := rmMeta.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) err := rmMeta.filesystem.DeleteMetadataByName(targetPath, attr) diff --git a/cmd/subcmd/rmticket.go b/cmd/subcmd/rmticket.go index eb5f3d2..705c15a 100644 --- a/cmd/subcmd/rmticket.go +++ b/cmd/subcmd/rmticket.go @@ -72,7 +72,7 @@ func (rmTicket *RmTicketCommand) Process() error { } // Create a file system - rmTicket.account = commons.GetAccount() + rmTicket.account = commons.GetSessionConfig().ToIRODSAccount() rmTicket.filesystem, err = commons.GetIRODSFSClient(rmTicket.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) diff --git a/cmd/subcmd/svrinfo.go b/cmd/subcmd/svrinfo.go index af207a4..b4b2938 100644 --- a/cmd/subcmd/svrinfo.go +++ b/cmd/subcmd/svrinfo.go @@ -70,7 +70,7 @@ func (svrInfo *SvrInfoCommand) Process() error { } // Create a file system - svrInfo.account = commons.GetAccount() + svrInfo.account = commons.GetSessionConfig().ToIRODSAccount() svrInfo.filesystem, err = commons.GetIRODSFSClient(svrInfo.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) diff --git a/cmd/subcmd/touch.go b/cmd/subcmd/touch.go index b20efb6..d92e3fc 100644 --- a/cmd/subcmd/touch.go +++ b/cmd/subcmd/touch.go @@ -77,7 +77,7 @@ func (touch *TouchCommand) Process() error { } // Create a file system - touch.account = commons.GetAccount() + touch.account = commons.GetSessionConfig().ToIRODSAccount() touch.filesystem, err = commons.GetIRODSFSClient(touch.account) if err != nil { return xerrors.Errorf("failed to get iRODS FS Client: %w", err) @@ -98,7 +98,7 @@ func (touch *TouchCommand) Process() error { func (touch *TouchCommand) touchOne(targetPath string) error { cwd := commons.GetCWD() home := commons.GetHomeDir() - zone := commons.GetZone() + zone := touch.account.ClientZone targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath) err := touch.filesystem.Touch(targetPath, "", touch.noCreateFlagValues.NoCreate) diff --git a/commons/bundle_transfer.go b/commons/bundle_transfer.go index 43c1cf1..51e8c10 100644 --- a/commons/bundle_transfer.go +++ b/commons/bundle_transfer.go @@ -164,6 +164,7 @@ func (bundle *Bundle) SetCompleted() { } type BundleTransferManager struct { + account *irodsclient_types.IRODSAccount filesystem *irodsclient_fs.FileSystem transferReportManager *TransferReportManager irodsDestPath string @@ -198,13 +199,14 @@ type BundleTransferManager struct { } // NewBundleTransferManager creates a new BundleTransferManager -func NewBundleTransferManager(fs *irodsclient_fs.FileSystem, transferReportManager *TransferReportManager, irodsDestPath string, localBundleRootPath string, minBundleFileNum int, maxBundleFileNum int, maxBundleFileSize int64, singleThreaded bool, uploadThreadNum int, redirectToResource bool, useIcat bool, localTempDirPath string, irodsTempDirPath string, noBulkReg bool, showProgress bool, showFullPath bool) *BundleTransferManager { +func NewBundleTransferManager(account *irodsclient_types.IRODSAccount, fs *irodsclient_fs.FileSystem, transferReportManager *TransferReportManager, irodsDestPath string, localBundleRootPath string, minBundleFileNum int, maxBundleFileNum int, maxBundleFileSize int64, singleThreaded bool, uploadThreadNum int, redirectToResource bool, useIcat bool, localTempDirPath string, irodsTempDirPath string, noBulkReg bool, showProgress bool, showFullPath bool) *BundleTransferManager { cwd := GetCWD() home := GetHomeDir() - zone := GetZone() + zone := account.ClientZone irodsDestPath = MakeIRODSPath(cwd, home, zone, irodsDestPath) manager := &BundleTransferManager{ + account: account, filesystem: fs, transferReportManager: transferReportManager, irodsDestPath: irodsDestPath, diff --git a/commons/commands.go b/commons/commands.go index 3b8421f..711651e 100644 --- a/commons/commands.go +++ b/commons/commands.go @@ -5,136 +5,63 @@ import ( "io" "os" "path" - "path/filepath" "strings" - irodsclient_icommands "github.com/cyverse/go-irodsclient/icommands" - irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" - "github.com/cyverse/go-irodsclient/irods/util" + irodsclient_config "github.com/cyverse/go-irodsclient/config" log "github.com/sirupsen/logrus" "golang.org/x/xerrors" - - "github.com/jedib0t/go-pretty/v6/table" ) var ( - environmentManager *irodsclient_icommands.ICommandsEnvironmentManager - appConfig *Config - account *irodsclient_types.IRODSAccount - - sessionID int + environmentManager *irodsclient_config.ICommandsEnvironmentManager ) -// GetEnvironmentManager returns environment manager -func GetEnvironmentManager() *irodsclient_icommands.ICommandsEnvironmentManager { - return environmentManager -} - -// GetConfig returns config -func GetConfig() *Config { - return appConfig -} - -// SetDefaultConfigIfEmpty sets default config if empty -func SetDefaultConfigIfEmpty() { - if environmentManager == nil { - iCommandsEnvMgr, _ := irodsclient_icommands.CreateIcommandsEnvironmentManager() - environmentManager = iCommandsEnvMgr - } - - if appConfig == nil { - appConfig = GetDefaultConfig() - setConfigToICommandsEnvMgr(environmentManager, appConfig) - } - - if environmentManager.Environment.LogLevel > 0 { - logLevel := getLogrusLogLevel(environmentManager.Environment.LogLevel) - log.SetLevel(logLevel) +// InitEnvironmentManager initializes envionment manager +func InitEnvironmentManager() error { + manager, err := irodsclient_config.NewICommandsEnvironmentManager() + if err != nil { + return err } - environmentManager.Load(sessionID) - setICommandsEnvMgrToConfig(appConfig, environmentManager) - - SyncAccount() -} - -// SetSessionID sets session id -func SetSessionID(id int) { - sessionID = id + environmentManager = manager + return nil } -// GetSessionID returns session id -func GetSessionID() int { - return sessionID +// GetEnvironmentManager returns environment manager +func GetEnvironmentManager() *irodsclient_config.ICommandsEnvironmentManager { + return environmentManager } -// SyncAccount syncs irods account -func SyncAccount() error { - newAccount, err := environmentManager.ToIRODSAccount() +// GetSessionConfig returns session configuration +func GetSessionConfig() *irodsclient_config.Config { + session, err := environmentManager.GetSessionConfig() if err != nil { - return xerrors.Errorf("failed to get account from iCommands Environment: %w", err) - } - - if len(appConfig.ClientUsername) > 0 { - newAccount.ClientUser = appConfig.ClientUsername - } - - if len(appConfig.DefaultResource) > 0 { - newAccount.DefaultResource = appConfig.DefaultResource - } - - if len(appConfig.DefaultHashScheme) > 0 { - newAccount.DefaultHashScheme = appConfig.DefaultHashScheme - } - - if len(appConfig.Ticket) > 0 { - newAccount.Ticket = appConfig.Ticket + return nil } - account = newAccount - return nil -} - -// GetAccount returns irods account -func GetAccount() *irodsclient_types.IRODSAccount { - return account + return session } // GetCWD returns current working directory func GetCWD() string { - session := environmentManager.Session - cwd := session.CurrentWorkingDir - - if len(cwd) == 0 { - env := environmentManager.Environment - cwd = env.CurrentWorkingDir + session, err := environmentManager.GetSessionConfig() + if err != nil { + return GetHomeDir() } - if len(cwd) > 0 { - if !strings.HasPrefix(cwd, "/") { + if len(session.CurrentWorkingDir) > 0 { + if !strings.HasPrefix(session.CurrentWorkingDir, "/") { // relative path from home - currentWorkingDir := path.Join(GetHomeDir(), cwd) + currentWorkingDir := path.Join(GetHomeDir(), session.CurrentWorkingDir) return path.Clean(currentWorkingDir) } - return path.Clean(cwd) + return path.Clean(session.CurrentWorkingDir) } return GetHomeDir() } -// GetZone returns zone -func GetZone() string { - env := environmentManager.Environment - return env.Zone -} - -// GetUsername returns username -func GetUsername() string { - env := environmentManager.Environment - return env.Username -} - // GetHomeDir returns home dir func GetHomeDir() string { env := environmentManager.Environment @@ -142,7 +69,7 @@ func GetHomeDir() string { return env.Home } - return fmt.Sprintf("/%s/home/%s", env.Zone, env.Username) + return fmt.Sprintf("/%s/home/%s", env.ClientZoneName, env.ClientUsername) } // SetCWD sets current workding directory @@ -160,8 +87,8 @@ func SetCWD(cwd string) error { session.CurrentWorkingDir = path.Clean(cwd) - logger.Debugf("save session to file - id %d", sessionID) - err := environmentManager.SaveSession(sessionID) + logger.Debugf("save session to file - id %d", environmentManager.PPID) + err := environmentManager.SaveSession() if err != nil { return xerrors.Errorf("failed to save session: %w", err) } @@ -170,19 +97,6 @@ func SetCWD(cwd string) error { // InputMissingFields inputs missing fields func InputMissingFields() (bool, error) { - if environmentManager == nil { - envMgr, err := irodsclient_icommands.CreateIcommandsEnvironmentManager() - if err != nil { - return false, xerrors.Errorf("failed to get new iCommands Environment: %w", err) - } - - environmentManager = envMgr - err = SyncAccount() - if err != nil { - return false, xerrors.Errorf("failed to get iCommands Environment: %w", err) - } - } - updated := false env := environmentManager.Environment @@ -200,10 +114,10 @@ func InputMissingFields() (bool, error) { updated = true } - if len(env.Zone) == 0 { - env.Zone = Input("iRODS Zone [iplant]") - if len(env.Zone) == 0 { - env.Zone = "iplant" + if len(env.ZoneName) == 0 { + env.ZoneName = Input("iRODS Zone [iplant]") + if len(env.ZoneName) == 0 { + env.ZoneName = "iplant" } updated = true @@ -214,32 +128,20 @@ func InputMissingFields() (bool, error) { updated = true } - password := environmentManager.Password - pamToken := environmentManager.PamToken + password := environmentManager.Environment.Password + pamToken := environmentManager.Environment.PAMToken if len(password) == 0 && len(pamToken) == 0 && env.Username != "anonymous" { - environmentManager.Password = InputPassword("iRODS Password") + environmentManager.Environment.Password = InputPassword("iRODS Password") updated = true } - err := SyncAccount() - if err != nil { - return updated, xerrors.Errorf("failed to get iCommands Environment: %w", err) - } + environmentManager.FixAuthConfiguration() return updated, nil } // InputMissingFieldsFromStdin inputs missing fields func InputMissingFieldsFromStdin() error { - if environmentManager == nil { - envMgr, err := irodsclient_icommands.CreateIcommandsEnvironmentManager() - if err != nil { - return xerrors.Errorf("failed to get new iCommands Environment: %w", err) - } - - environmentManager = envMgr - } - // read from stdin stdinBytes, err := io.ReadAll(os.Stdin) if err != nil { @@ -251,31 +153,19 @@ func InputMissingFieldsFromStdin() error { return xerrors.Errorf("failed to read missing config values: %w", err) } - env := environmentManager.Environment - env.Host = configTypeIn.Host - env.Port = configTypeIn.Port - env.Zone = configTypeIn.Zone - env.Username = configTypeIn.Username - environmentManager.Password = configTypeIn.Password + environmentManager.Environment.Host = configTypeIn.Host + environmentManager.Environment.Port = configTypeIn.Port + environmentManager.Environment.ZoneName = configTypeIn.ZoneName + environmentManager.Environment.Username = configTypeIn.Username + environmentManager.Environment.Password = configTypeIn.Password + + environmentManager.FixAuthConfiguration() return nil } // ReinputFields re-inputs fields func ReinputFields() (bool, error) { - if environmentManager == nil { - envMgr, err := irodsclient_icommands.CreateIcommandsEnvironmentManager() - if err != nil { - return false, xerrors.Errorf("failed to get new iCommands Environment: %w", err) - } - - environmentManager = envMgr - err = SyncAccount() - if err != nil { - return false, xerrors.Errorf("failed to get iCommands Environment: %w", err) - } - } - updated := false env := environmentManager.Environment @@ -299,13 +189,13 @@ func ReinputFields() (bool, error) { updated = true } - if len(env.Zone) == 0 { - env.Zone = "iplant" // default + if len(env.ZoneName) == 0 { + env.ZoneName = "iplant" // default } - newZone := Input(fmt.Sprintf("iRODS Zone [%s]", env.Zone)) - if len(newZone) > 0 && newZone != env.Zone { - env.Zone = newZone + newZone := Input(fmt.Sprintf("iRODS Zone [%s]", env.ZoneName)) + if len(newZone) > 0 && newZone != env.ZoneName { + env.ZoneName = newZone updated = true } @@ -330,396 +220,9 @@ func ReinputFields() (bool, error) { newPassword := InputPassword("iRODS Password") updated = true - environmentManager.Password = newPassword + environmentManager.Environment.Password = newPassword - err := SyncAccount() - if err != nil { - return updated, xerrors.Errorf("failed to get iCommands Environment: %w", err) - } + environmentManager.FixAuthConfiguration() return updated, nil } - -func isICommandsEnvDir(dirPath string) bool { - realDirPath, err := ResolveSymlink(dirPath) - if err != nil { - return false - } - - st, err := os.Stat(realDirPath) - if err != nil { - return false - } - - if !st.IsDir() { - return false - } - - entries, err := os.ReadDir(dirPath) - if err != nil { - return false - } - - for _, entry := range entries { - if !entry.IsDir() { - if strings.HasPrefix(entry.Name(), "irods_environment.json.") { - return true - } else if entry.Name() == "irods_environment.json" { - return true - } else if entry.Name() == ".irodsA" { - return true - } - } - } - - return false -} - -func isYAMLFile(filePath string) bool { - st, err := os.Stat(filePath) - if err != nil { - return false - } - - if st.IsDir() { - return false - } - - ext := filepath.Ext(filePath) - return ext == ".yaml" || ext == ".yml" -} - -func setConfigToICommandsEnvMgr(envManager *irodsclient_icommands.ICommandsEnvironmentManager, config *Config) { - envManager.Environment.CurrentWorkingDir = config.CurrentWorkingDir - envManager.Environment.Host = config.Host - envManager.Environment.Port = config.Port - envManager.Environment.Username = config.Username - envManager.Environment.Zone = config.Zone - envManager.Environment.DefaultResource = config.DefaultResource - envManager.Environment.DefaultHashScheme = config.DefaultHashScheme - envManager.Environment.LogLevel = config.LogLevel - envManager.Environment.AuthenticationScheme = config.AuthenticationScheme - envManager.Environment.ClientServerNegotiation = config.ClientServerNegotiation - envManager.Environment.ClientServerPolicy = config.ClientServerPolicy - envManager.Environment.SSLCACertificateFile = config.SSLCACertificateFile - envManager.Environment.SSLCACertificatePath = config.SSLCACertificatePath - envManager.Environment.SSLVerifyServer = config.SSLVerifyServer - envManager.Environment.EncryptionKeySize = config.EncryptionKeySize - envManager.Environment.EncryptionAlgorithm = config.EncryptionAlgorithm - envManager.Environment.EncryptionSaltSize = config.EncryptionSaltSize - envManager.Environment.EncryptionNumHashRounds = config.EncryptionNumHashRounds - - envManager.Password = config.Password -} - -func overwriteConfigToICommandsEnvMgr(envManager *irodsclient_icommands.ICommandsEnvironmentManager, config *Config) { - if len(config.CurrentWorkingDir) > 0 { - envManager.Environment.CurrentWorkingDir = config.CurrentWorkingDir - } - - if len(config.Host) > 0 { - envManager.Environment.Host = config.Host - } - - if config.Port > 0 { - envManager.Environment.Port = config.Port - } - - if len(config.Username) > 0 { - envManager.Environment.Username = config.Username - } - - if len(config.Zone) > 0 { - envManager.Environment.Zone = config.Zone - } - - if len(config.DefaultResource) > 0 { - envManager.Environment.DefaultResource = config.DefaultResource - } - - if len(config.DefaultHashScheme) > 0 { - envManager.Environment.DefaultHashScheme = config.DefaultHashScheme - } - - if config.LogLevel > 0 { - envManager.Environment.LogLevel = config.LogLevel - } - - if len(config.AuthenticationScheme) > 0 { - envManager.Environment.AuthenticationScheme = config.AuthenticationScheme - } - - if len(config.ClientServerNegotiation) > 0 { - envManager.Environment.ClientServerNegotiation = config.ClientServerNegotiation - } - - if len(config.ClientServerPolicy) > 0 { - envManager.Environment.ClientServerPolicy = config.ClientServerPolicy - } - - if len(config.SSLCACertificateFile) > 0 { - envManager.Environment.SSLCACertificateFile = config.SSLCACertificateFile - } - - if len(config.SSLCACertificatePath) > 0 { - envManager.Environment.SSLCACertificatePath = config.SSLCACertificatePath - } - - if len(config.SSLVerifyServer) > 0 { - envManager.Environment.SSLVerifyServer = config.SSLVerifyServer - } - - if config.EncryptionKeySize > 0 { - envManager.Environment.EncryptionKeySize = config.EncryptionKeySize - } - - if len(config.EncryptionAlgorithm) > 0 { - envManager.Environment.EncryptionAlgorithm = config.EncryptionAlgorithm - } - - if config.EncryptionSaltSize > 0 { - envManager.Environment.EncryptionSaltSize = config.EncryptionSaltSize - } - - if config.EncryptionNumHashRounds > 0 { - envManager.Environment.EncryptionNumHashRounds = config.EncryptionNumHashRounds - } - - if len(config.Password) > 0 { - envManager.Password = config.Password - } -} - -func setICommandsEnvMgrToConfig(config *Config, envManager *irodsclient_icommands.ICommandsEnvironmentManager) { - config.CurrentWorkingDir = envManager.Environment.CurrentWorkingDir - config.Host = envManager.Environment.Host - config.Port = envManager.Environment.Port - config.Username = envManager.Environment.Username - config.Zone = envManager.Environment.Zone - config.DefaultResource = envManager.Environment.DefaultResource - config.DefaultHashScheme = envManager.Environment.DefaultHashScheme - config.LogLevel = envManager.Environment.LogLevel - config.AuthenticationScheme = envManager.Environment.AuthenticationScheme - config.ClientServerNegotiation = envManager.Environment.ClientServerNegotiation - config.ClientServerPolicy = envManager.Environment.ClientServerPolicy - config.SSLCACertificateFile = envManager.Environment.SSLCACertificateFile - config.SSLCACertificatePath = envManager.Environment.SSLCACertificatePath - config.SSLVerifyServer = envManager.Environment.SSLVerifyServer - config.EncryptionKeySize = envManager.Environment.EncryptionKeySize - config.EncryptionAlgorithm = envManager.Environment.EncryptionAlgorithm - config.EncryptionSaltSize = envManager.Environment.EncryptionSaltSize - config.EncryptionNumHashRounds = envManager.Environment.EncryptionNumHashRounds - - config.Password = envManager.Password -} - -func getLogrusLogLevel(irodsLogLevel int) log.Level { - switch irodsLogLevel { - case 0: - return log.PanicLevel - case 1: - return log.FatalLevel - case 2, 3: - return log.ErrorLevel - case 4, 5, 6: - return log.WarnLevel - case 7: - return log.InfoLevel - case 8: - return log.DebugLevel - case 9, 10: - return log.TraceLevel - } - - if irodsLogLevel < 0 { - return log.PanicLevel - } - - return log.TraceLevel -} - -func LoadConfigFromFile(configPath string) error { - logger := log.WithFields(log.Fields{ - "package": "commons", - "function": "LoadConfigFromFile", - }) - - configPath, err := ExpandHomeDir(configPath) - if err != nil { - return xerrors.Errorf("failed to expand home directory for %q: %w", configPath, err) - } - - configPath, err = filepath.Abs(configPath) - if err != nil { - return xerrors.Errorf("failed to compute absolute path for %q: %w", configPath, err) - } - - logger.Debugf("reading config path %q", configPath) - // check if it is a file or a dir - _, err = os.Stat(configPath) - if err != nil { - if os.IsNotExist(err) { - return irodsclient_types.NewFileNotFoundError(configPath) - } - return xerrors.Errorf("failed to stat %q: %w", configPath, err) - } - - if isYAMLFile(configPath) { - logger.Debugf("reading gocommands YAML config file %q", configPath) - - iCommandsEnvMgr, err := irodsclient_icommands.CreateIcommandsEnvironmentManager() - if err != nil { - return xerrors.Errorf("failed to create iCommands Environment: %w", err) - } - - err = iCommandsEnvMgr.SetEnvironmentFilePath(configPath) - if err != nil { - return xerrors.Errorf("failed to set environment file path %q: %w", configPath, err) - } - - // read session - sessionFilePath := iCommandsEnvMgr.GetSessionFilePath(sessionID) - if util.ExistFile(sessionFilePath) { - session, err := irodsclient_icommands.CreateICommandsEnvironmentFromFile(sessionFilePath) - if err != nil { - return xerrors.Errorf("failed to create icommands environment from file %q: %w", sessionFilePath, err) - } - - iCommandsEnvMgr.Session = session - } - - // load from YAML - yjBytes, err := os.ReadFile(configPath) - if err != nil { - return xerrors.Errorf("failed to read file %q: %w", configPath, err) - } - - defaultConfig := GetDefaultConfig() - config, err := NewConfigFromYAML(defaultConfig, yjBytes) - if err != nil { - return xerrors.Errorf("failed to read config from YAML: %w", err) - } - - setConfigToICommandsEnvMgr(iCommandsEnvMgr, config) - - if iCommandsEnvMgr.Environment.LogLevel > 0 { - logLevel := getLogrusLogLevel(iCommandsEnvMgr.Environment.LogLevel) - log.SetLevel(logLevel) - } - - environmentManager = iCommandsEnvMgr - appConfig = config - - err = SyncAccount() - if err != nil { - return xerrors.Errorf("failed to sync account: %w", err) - } - - return nil - } - - // icommands compatible - configFilePath := configPath - if isICommandsEnvDir(configPath) { - configFilePath = filepath.Join(configPath, "irods_environment.json") - } - - logger.Debugf("reading icommands environment file %q", configFilePath) - - iCommandsEnvMgr, err := irodsclient_icommands.CreateIcommandsEnvironmentManager() - if err != nil { - return xerrors.Errorf("failed to create new iCommands Environment: %w", err) - } - - err = iCommandsEnvMgr.SetEnvironmentFilePath(configFilePath) - if err != nil { - return xerrors.Errorf("failed to set iCommands Environment file %q: %w", configFilePath, err) - } - - err = iCommandsEnvMgr.Load(sessionID) - if err != nil { - return xerrors.Errorf("failed to read iCommands Environment: %w", err) - } - - config := GetDefaultConfig() - - setICommandsEnvMgrToConfig(config, iCommandsEnvMgr) - - if iCommandsEnvMgr.Environment.LogLevel > 0 { - logLevel := getLogrusLogLevel(iCommandsEnvMgr.Environment.LogLevel) - log.SetLevel(logLevel) - } - - environmentManager = iCommandsEnvMgr - appConfig = config - - err = SyncAccount() - if err != nil { - return xerrors.Errorf("failed to sync account: %w", err) - } - - return nil -} - -// LoadAndOverwriteConfigFromEnv loads config from env and overwrites to existing env -func LoadAndOverwriteConfigFromEnv() error { - logger := log.WithFields(log.Fields{ - "package": "commons", - "function": "LoadAndOverwriteConfigFromEnv", - }) - - logger.Debug("reading config from environment variables") - - config, err := NewConfigFromENV() - if err != nil { - return xerrors.Errorf("failed to get new iCommands Environment: %w", err) - } - - overwriteConfigToICommandsEnvMgr(environmentManager, config) - - if environmentManager.Environment.LogLevel > 0 { - logLevel := getLogrusLogLevel(environmentManager.Environment.LogLevel) - log.SetLevel(logLevel) - } - - setICommandsEnvMgrToConfig(appConfig, environmentManager) - - SyncAccount() - - return nil -} - -func PrintAccount() error { - envMgr := GetEnvironmentManager() - if envMgr == nil { - return xerrors.Errorf("environment is not set") - } - - t := table.NewWriter() - t.SetOutputMirror(os.Stdout) - - t.AppendRows([]table.Row{ - { - "iRODS Host", - envMgr.Environment.Host, - }, - { - "iRODS Port", - envMgr.Environment.Port, - }, - { - "iRODS Zone", - envMgr.Environment.Zone, - }, - { - "iRODS Username", - envMgr.Environment.Username, - }, - { - "iRODS Authentication Scheme", - envMgr.Environment.AuthenticationScheme, - }, - }, table.RowConfig{}) - t.Render() - return nil -} diff --git a/commons/config.go b/commons/config.go index 8ff7b91..c6942dd 100644 --- a/commons/config.go +++ b/commons/config.go @@ -1,6 +1,9 @@ package commons import ( + "os" + "path/filepath" + irodsclient_types "github.com/cyverse/go-irodsclient/irods/types" "golang.org/x/xerrors" @@ -13,7 +16,7 @@ const ( HashSchemeDefault string = irodsclient_types.HashSchemeDefault AuthenticationSchemeDefault string = string(irodsclient_types.AuthSchemeNative) - ClientServerPolicyDefault string = string(irodsclient_types.CSNegotiationRequireTCP) + ClientServerPolicyDefault string = string(irodsclient_types.CSNegotiationPolicyRequestTCP) EncryptionKeySizeDefault int = 32 EncryptionAlgorithmDefault string = "AES-256-CBC" SaltSizeDefault int = 8 @@ -27,7 +30,7 @@ type Config struct { Port int `yaml:"irods_port,omitempty" envconfig:"IRODS_PORT"` Username string `yaml:"irods_user_name,omitempty" envconfig:"IRODS_USER_NAME"` ClientUsername string `yaml:"irods_client_user_name,omitempty" envconfig:"IRODS_CLIENT_USER_NAME"` - Zone string `yaml:"irods_zone_name,omitempty" envconfig:"IRODS_ZONE_NAME"` + ZoneName string `yaml:"irods_zone_name,omitempty" envconfig:"IRODS_ZONE_NAME"` DefaultResource string `yaml:"irods_default_resource,omitempty" envconfig:"IRODS_DEFAULT_RESOURCE"` DefaultHashScheme string `yaml:"irods_default_hash_scheme,omitempty" envconfig:"IRODS_DEFAULT_HASH_SCHEME"` LogLevel int `yaml:"irods_log_level,omitempty" envconfig:"IRODS_LOG_LEVEL"` @@ -93,10 +96,24 @@ func GetDefaultIRODSConfigPath() string { return irodsConfigPath } +func IsYAMLFile(filePath string) bool { + st, err := os.Stat(filePath) + if err != nil { + return false + } + + if st.IsDir() { + return false + } + + ext := filepath.Ext(filePath) + return ext == ".yaml" || ext == ".yml" +} + type ConfigTypeIn struct { Host string `yaml:"irods_host,omitempty"` Port int `yaml:"irods_port,omitempty"` - Zone string `yaml:"irods_zone_name,omitempty"` + ZoneName string `yaml:"irods_zone_name,omitempty"` Username string `yaml:"irods_user_name,omitempty"` Password string `yaml:"irods_user_password,omitempty"` } diff --git a/commons/retry.go b/commons/retry.go index 1a28969..a8813dd 100644 --- a/commons/retry.go +++ b/commons/retry.go @@ -52,9 +52,9 @@ func runChild() error { configTypeIn := &ConfigTypeIn{ Host: env.Host, Port: env.Port, - Zone: env.Zone, + ZoneName: env.ZoneName, Username: env.Username, - Password: envManager.Password, + Password: envManager.Environment.Password, } configTypeInYamlBytes, err := configTypeIn.ToYAML() diff --git a/commons/staging.go b/commons/staging.go index 2e7a678..b65ca96 100644 --- a/commons/staging.go +++ b/commons/staging.go @@ -63,6 +63,8 @@ func IsStagingDirInTargetPath(stagingPath string) bool { } func IsSafeStagingDir(stagingPath string) error { + sessionConfig := GetSessionConfig() + dirParts := strings.Split(stagingPath[1:], "/") dirDepth := len(dirParts) @@ -72,7 +74,7 @@ func IsSafeStagingDir(stagingPath string) error { } // zone/home/user OR zone/home/shared (public) - if dirParts[0] != GetZone() { + if dirParts[0] != sessionConfig.ClientZoneName { return xerrors.Errorf("staging path %q is not safe, not in the correct zone", stagingPath) } @@ -80,7 +82,7 @@ func IsSafeStagingDir(stagingPath string) error { return xerrors.Errorf("staging path %q is not safe", stagingPath) } - if dirParts[2] == GetUsername() { + if dirParts[2] == sessionConfig.ClientUsername { if dirDepth <= 3 { // /zone/home/user return xerrors.Errorf("staging path %q is not safe!", stagingPath) diff --git a/commons/transfer_report.go b/commons/transfer_report.go index 77fbf00..b230f3b 100644 --- a/commons/transfer_report.go +++ b/commons/transfer_report.go @@ -37,13 +37,14 @@ type TransferReportFile struct { StartAt time.Time `json:"start_time"` EndAt time.Time `json:"end_at"` - SourcePath string `json:"source_path"` - DestPath string `json:"dest_path"` - ChecksumAlgorithm string `json:"checksum_algorithm"` - SourceSize int64 `json:"source_size"` - SourceChecksum string `json:"source_checksum"` - DestSize int64 `json:"dest_size"` - DestChecksum string `json:"dest_checksum"` + SourcePath string `json:"source_path"` + DestPath string `json:"dest_path"` + SourceSize int64 `json:"source_size"` + SourceChecksumAlgorithm string `json:"source_checksum_algorithm"` + SourceChecksum string `json:"source_checksum"` + DestSize int64 `json:"dest_size"` + DestChecksumAlgorithm string `json:"dest_checksum_algorithm"` + DestChecksum string `json:"dest_checksum"` Error error `json:"error,omitempty"` Notes []string `json:"notes"` // additional notes @@ -76,17 +77,18 @@ func NewTransferReportFileFromTransferResult(result *irodsclient_fs.FileTransfer StartAt: result.StartTime, EndAt: result.EndTime, - SourcePath: result.IRODSPath, - SourceSize: result.IRODSSize, - SourceChecksum: hex.EncodeToString(result.IRODSCheckSum), + SourcePath: result.IRODSPath, + SourceSize: result.IRODSSize, + SourceChecksumAlgorithm: string(result.IRODSCheckSumAlgorithm), + SourceChecksum: hex.EncodeToString(result.IRODSCheckSum), - DestPath: result.LocalPath, - DestSize: result.LocalSize, - DestChecksum: hex.EncodeToString(result.LocalCheckSum), + DestPath: result.LocalPath, + DestSize: result.LocalSize, + DestChecksumAlgorithm: string(result.LocalCheckSumAlgorithm), + DestChecksum: hex.EncodeToString(result.LocalCheckSum), - ChecksumAlgorithm: string(result.CheckSumAlgorithm), - Error: err, - Notes: notes, + Error: err, + Notes: notes, }, nil } else if method == TransferMethodPut || method == TransferMethodBput { // put @@ -96,17 +98,18 @@ func NewTransferReportFileFromTransferResult(result *irodsclient_fs.FileTransfer StartAt: result.StartTime, EndAt: result.EndTime, - SourcePath: result.LocalPath, - SourceSize: result.LocalSize, - SourceChecksum: hex.EncodeToString(result.LocalCheckSum), + SourcePath: result.LocalPath, + SourceSize: result.LocalSize, + SourceChecksumAlgorithm: string(result.LocalCheckSumAlgorithm), + SourceChecksum: hex.EncodeToString(result.LocalCheckSum), - DestPath: result.IRODSPath, - DestSize: result.IRODSSize, - DestChecksum: hex.EncodeToString(result.IRODSCheckSum), + DestPath: result.IRODSPath, + DestSize: result.IRODSSize, + DestChecksumAlgorithm: string(result.IRODSCheckSumAlgorithm), + DestChecksum: hex.EncodeToString(result.IRODSCheckSum), - ChecksumAlgorithm: string(result.CheckSumAlgorithm), - Error: err, - Notes: notes, + Error: err, + Notes: notes, }, nil } else { return nil, xerrors.Errorf("unknown method %q", method) diff --git a/go.mod b/go.mod index 6bf0775..0d2a40f 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/creativeprojects/go-selfupdate v1.0.1 - github.com/cyverse/go-irodsclient v0.15.1 + github.com/cyverse/go-irodsclient v0.15.3 github.com/dustin/go-humanize v1.0.1 github.com/gliderlabs/ssh v0.3.5 github.com/jedib0t/go-pretty/v6 v6.3.1 @@ -49,4 +49,5 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 1bddb7a..864984f 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creativeprojects/go-selfupdate v1.0.1 h1:5Un4MTv4puCR5GBgkDLC14J72fljGuMC60E63cFGq1o= github.com/creativeprojects/go-selfupdate v1.0.1/go.mod h1:nm7AWUJfrfYt/SB97NAcMhR0KEpPqlrVHXkWFti+ezw= -github.com/cyverse/go-irodsclient v0.15.1 h1:SpTP5dpcp0zko/5R4niK3WZOayl1Ptj062llxywm1Y0= -github.com/cyverse/go-irodsclient v0.15.1/go.mod h1:eBXha3cwfrM0p1ijYVqsrLJQHpRwTfpA4c5dKCQsQFc= +github.com/cyverse/go-irodsclient v0.15.3 h1:PyZOYniwXUNLtO2zEzP+u+yBQrI0ayusxctxNOoL+QI= +github.com/cyverse/go-irodsclient v0.15.3/go.mod h1:NN+PxHfLDUmsqfqSY84JfmqXS4EYiuiNW6ti6oPGCgk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -150,6 +150,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=