Skip to content

Commit

Permalink
Merge pull request #48 from cyverse/transfer_report
Browse files Browse the repository at this point in the history
Transfer report
  • Loading branch information
iychoi authored Aug 22, 2024
2 parents 63356c6 + 0d05645 commit 6bd9342
Show file tree
Hide file tree
Showing 56 changed files with 5,553 additions and 2,937 deletions.
2 changes: 1 addition & 1 deletion VERSION.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
v0.9.10
v0.9.11
57 changes: 20 additions & 37 deletions cmd/flag/bundle_transfer.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,54 +8,37 @@ import (
"github.com/spf13/cobra"
)

type BundleTempFlagValues struct {
LocalTempPath string
IRODSTempPath string
}

type BundleClearFlagVlaues struct {
Clear bool
}

type BundleConfigFlagValues struct {
type BundleTransferFlagValues struct {
LocalTempPath string
IRODSTempPath string
ClearOld bool
MinFileNum int
MaxFileNum int
MaxFileSize int64
NoBulkRegistration bool
maxFileSizeInput string
}

var (
bundleTempFlagValues BundleTempFlagValues
bundleClearFlagValues BundleClearFlagVlaues
bundleConfigFlagValues BundleConfigFlagValues
bundleTransferFlagValues BundleTransferFlagValues
)

func SetBundleTempFlags(command *cobra.Command) {
command.Flags().StringVar(&bundleTempFlagValues.LocalTempPath, "local_temp", os.TempDir(), "Specify local temp directory path to create bundle files")
command.Flags().StringVar(&bundleTempFlagValues.IRODSTempPath, "irods_temp", "", "Specify iRODS temp collection path to upload bundle files to")
}

func GetBundleTempFlagValues() *BundleTempFlagValues {
return &bundleTempFlagValues
}

func SetBundleClearFlags(command *cobra.Command) {
command.Flags().BoolVar(&bundleClearFlagValues.Clear, "clear", false, "Clear stale bundle files")
}

func GetBundleClearFlagValues() *BundleClearFlagVlaues {
return &bundleClearFlagValues
}
func SetBundleTransferFlags(command *cobra.Command, displayTransferConfig bool) {
command.Flags().StringVar(&bundleTransferFlagValues.LocalTempPath, "local_temp", os.TempDir(), "Specify local temp directory path to create bundle files")
command.Flags().StringVar(&bundleTransferFlagValues.IRODSTempPath, "irods_temp", "", "Specify iRODS temp collection path to upload bundle files to")

func SetBundleConfigFlags(command *cobra.Command) {
command.Flags().IntVar(&bundleConfigFlagValues.MaxFileNum, "max_file_num", commons.MaxBundleFileNumDefault, "Specify max file number in a bundle file")
command.Flags().StringVar(&bundleConfigFlagValues.maxFileSizeInput, "max_file_size", strconv.FormatInt(commons.MaxBundleFileSizeDefault, 10), "Specify max file size of a bundle file")
command.Flags().BoolVar(&bundleConfigFlagValues.NoBulkRegistration, "no_bulk_reg", false, "Disable bulk registration")
if displayTransferConfig {
command.Flags().BoolVar(&bundleTransferFlagValues.ClearOld, "clear", false, "Clear stale bundle files")
command.Flags().IntVar(&bundleTransferFlagValues.MinFileNum, "min_file_num", commons.MinBundleFileNumDefault, "Specify min file number in a bundle file")
command.Flags().IntVar(&bundleTransferFlagValues.MaxFileNum, "max_file_num", commons.MaxBundleFileNumDefault, "Specify max file number in a bundle file")
command.Flags().StringVar(&bundleTransferFlagValues.maxFileSizeInput, "max_file_size", strconv.FormatInt(commons.MaxBundleFileSizeDefault, 10), "Specify max file size of a bundle file")
command.Flags().BoolVar(&bundleTransferFlagValues.NoBulkRegistration, "no_bulk_reg", false, "Disable bulk registration")
}
}

func GetBundleConfigFlagValues() *BundleConfigFlagValues {
size, _ := commons.ParseSize(bundleConfigFlagValues.maxFileSizeInput)
bundleConfigFlagValues.MaxFileSize = size
func GetBundleTransferFlagValues() *BundleTransferFlagValues {
size, _ := commons.ParseSize(bundleTransferFlagValues.maxFileSizeInput)
bundleTransferFlagValues.MaxFileSize = size

return &bundleConfigFlagValues
return &bundleTransferFlagValues
}
10 changes: 5 additions & 5 deletions cmd/flag/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ var (
)

func SetCommonFlags(command *cobra.Command, noResource bool) {
command.Flags().StringVarP(&commonFlagValues.ConfigFilePath, "config", "c", "", fmt.Sprintf("Set config file or dir (default \"%s\")", commons.GetDefaultIRODSConfigPath()))
command.Flags().StringVarP(&commonFlagValues.ConfigFilePath, "config", "c", "", fmt.Sprintf("Set config file or directory (default %q)", commons.GetDefaultIRODSConfigPath()))
command.Flags().BoolVarP(&commonFlagValues.ShowVersion, "version", "v", false, "Print version")
command.Flags().BoolVarP(&commonFlagValues.ShowHelp, "help", "h", false, "Print help")
command.Flags().BoolVarP(&commonFlagValues.DebugMode, "debug", "d", false, "Enable debug mode")
Expand Down Expand Up @@ -116,7 +116,7 @@ func ProcessCommonFlags(command *cobra.Command) (bool, error) {
// user defined config file
err := commons.LoadConfigFromFile(myCommonFlagValues.ConfigFilePath)
if err != nil {
return false, xerrors.Errorf("failed to load config from file %s: %w", myCommonFlagValues.ConfigFilePath, err) // stop here
return false, xerrors.Errorf("failed to load config from file %q: %w", myCommonFlagValues.ConfigFilePath, err) // stop here
}

readConfig = true
Expand All @@ -127,7 +127,7 @@ func ProcessCommonFlags(command *cobra.Command) (bool, error) {
if len(irodsEnvironmentFileEnvVal) > 0 {
err := commons.LoadConfigFromFile(irodsEnvironmentFileEnvVal)
if err != nil {
return false, xerrors.Errorf("failed to load config file %s: %w", irodsEnvironmentFileEnvVal, err) // stop here
return false, xerrors.Errorf("failed to load config file %q: %w", irodsEnvironmentFileEnvVal, err) // stop here
}

readConfig = true
Expand Down Expand Up @@ -176,7 +176,7 @@ func ProcessCommonFlags(command *cobra.Command) (bool, error) {
syncAccount := false
if myCommonFlagValues.ResourceUpdated {
appConfig.DefaultResource = myCommonFlagValues.Resource
logger.Debugf("use default resource server - %s", appConfig.DefaultResource)
logger.Debugf("use default resource server %q", appConfig.DefaultResource)
syncAccount = true
}

Expand All @@ -196,6 +196,6 @@ func printVersion() error {
return xerrors.Errorf("failed to get version json: %w", err)
}

fmt.Println(info)
commons.Println(info)
return nil
}
2 changes: 1 addition & 1 deletion cmd/flag/dry_run.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ var (
)

func SetDryRunFlags(command *cobra.Command) {
command.Flags().BoolVar(&dryRunFlagValues.DryRun, "dry_run", false, "Do not actually perform changes")
command.Flags().BoolVar(&dryRunFlagValues.DryRun, "dry_run", false, "Do not actually change")
}

func GetDryRunFlagValues() *DryRunFlagValues {
Expand Down
4 changes: 1 addition & 3 deletions cmd/flag/encryption.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ type EncryptionFlagValues struct {
type DecryptionFlagValues struct {
Decryption bool
NoDecryption bool
IgnoreMeta bool
Key string
PrivateKeyPath string
TempPath string
Expand All @@ -43,9 +42,8 @@ func SetEncryptionFlags(command *cobra.Command) {
}

func SetDecryptionFlags(command *cobra.Command) {
command.Flags().BoolVar(&decryptionFlagValues.Decryption, "decrypt", false, "Decrypt files")
command.Flags().BoolVar(&decryptionFlagValues.Decryption, "decrypt", true, "Decrypt files")
command.Flags().BoolVar(&decryptionFlagValues.NoDecryption, "no_decrypt", false, "Disable decryption forcefully")
command.Flags().BoolVar(&decryptionFlagValues.IgnoreMeta, "ignore_meta", false, "Ignore decryption config via metadata")
command.Flags().StringVar(&decryptionFlagValues.Key, "decrypt_key", "", "Decryption key for 'winscp' and 'pgp' mode")
command.Flags().StringVar(&decryptionFlagValues.PrivateKeyPath, "decrypt_priv_key", commons.GetDefaultPrivateKeyPath(), "Decryption private key for 'ssh' mode")
command.Flags().StringVar(&decryptionFlagValues.TempPath, "decrypt_temp", os.TempDir(), "Specify temp directory path for decrypting files")
Expand Down
14 changes: 12 additions & 2 deletions cmd/flag/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,25 @@ import (
)

type SyncFlagValues struct {
Delete bool
Delete bool
BulkUpload bool
Sync bool
}

var (
syncFlagValues SyncFlagValues
)

func SetSyncFlags(command *cobra.Command) {
func SetSyncFlags(command *cobra.Command, addPutOptionFlag bool) {
command.Flags().BoolVar(&syncFlagValues.Delete, "delete", false, "Delete extra files in dest dir")

if addPutOptionFlag {
command.Flags().BoolVar(&syncFlagValues.BulkUpload, "bulk_upload", false, "Use bulk upload")
}

// this is hidden
command.Flags().BoolVar(&syncFlagValues.Sync, "sync", false, "Set this for sync")
command.Flags().MarkHidden("sync")
}

func GetSyncFlagValues() *SyncFlagValues {
Expand Down
33 changes: 33 additions & 0 deletions cmd/flag/transfer_report.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package flag

import (
"github.com/spf13/cobra"
)

type TransferReportFlagValues struct {
ReportPath string
Report bool
ReportToStdout bool
}

var (
transferReportFlagValues TransferReportFlagValues
)

func SetTransferReportFlags(command *cobra.Command) {
command.Flags().StringVar(&transferReportFlagValues.ReportPath, "report", "", "Create a transfer report, give path to create a file, empty string or '-' will output to stdout")
}

func GetTransferReportFlagValues(command *cobra.Command) *TransferReportFlagValues {
if command.Flags().Changed("report") {
transferReportFlagValues.Report = true
}

transferReportFlagValues.ReportToStdout = false

if transferReportFlagValues.ReportPath == "-" || len(transferReportFlagValues.ReportPath) == 0 {
transferReportFlagValues.ReportToStdout = true
}

return &transferReportFlagValues
}
55 changes: 33 additions & 22 deletions cmd/gocmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@ package main

import (
"errors"
"fmt"
"os"

irodsclient_types "github.com/cyverse/go-irodsclient/irods/types"
"github.com/cyverse/gocommands/cmd/flag"
"github.com/cyverse/gocommands/cmd/subcmd"
"github.com/cyverse/gocommands/commons"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -54,12 +54,15 @@ func processCommand(command *cobra.Command, args []string) error {
}

func main() {
commons.InitTerminalOutput()

log.SetFormatter(&log.TextFormatter{
TimestampFormat: "2006-01-02 15:04:05.000",
FullTimestamp: true,
})

log.SetLevel(log.FatalLevel)
log.SetOutput(commons.GetTerminalWriter())

logger := log.WithFields(log.Fields{
"package": "main",
Expand Down Expand Up @@ -105,75 +108,83 @@ func main() {
logger.Errorf("%+v", err)

if os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "File or dir not found!\n")
commons.PrintErrorf("File or directory not found!\n")
} else if irodsclient_types.IsConnectionConfigError(err) {
var connectionConfigError *irodsclient_types.ConnectionConfigError
if errors.As(err, &connectionConfigError) {
fmt.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server (host: '%s', port: '%d')!\n", connectionConfigError.Config.Host, connectionConfigError.Config.Port)
commons.PrintErrorf("Failed to establish a connection to iRODS server (host: %q, port: %d)!\n", connectionConfigError.Config.Host, connectionConfigError.Config.Port)
} else {
fmt.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server!\n")
commons.PrintErrorf("Failed to establish a connection to iRODS server!\n")
}
} else if irodsclient_types.IsConnectionError(err) {
fmt.Fprintf(os.Stderr, "Failed to establish a connection to iRODS server!\n")
commons.PrintErrorf("Failed to establish a connection to iRODS server!\n")
} else if irodsclient_types.IsConnectionPoolFullError(err) {
var connectionPoolFullError *irodsclient_types.ConnectionPoolFullError
if errors.As(err, &connectionPoolFullError) {
fmt.Fprintf(os.Stderr, "Failed to establish a new connection to iRODS server as connection pool is full (occupied: %d, max: %d)!\n", connectionPoolFullError.Occupied, connectionPoolFullError.Max)
commons.PrintErrorf("Failed to establish a new connection to iRODS server as connection pool is full (occupied: %d, max: %d)!\n", connectionPoolFullError.Occupied, connectionPoolFullError.Max)
} else {
fmt.Fprintf(os.Stderr, "Failed to establish a new connection to iRODS server as connection pool is full!\n")
commons.PrintErrorf("Failed to establish a new connection to iRODS server as connection pool is full!\n")
}
} else if irodsclient_types.IsAuthError(err) {
var authError *irodsclient_types.AuthError
if errors.As(err, &authError) {
fmt.Fprintf(os.Stderr, "Authentication failed (auth scheme: '%s', username: '%s', zone: '%s')!\n", authError.Config.AuthenticationScheme, authError.Config.ClientUser, authError.Config.ClientZone)
commons.PrintErrorf("Authentication failed (auth scheme: %q, username: %q, zone: %q)!\n", authError.Config.AuthenticationScheme, authError.Config.ClientUser, authError.Config.ClientZone)
} else {
fmt.Fprintf(os.Stderr, "Authentication failed!\n")
commons.PrintErrorf("Authentication failed!\n")
}
} else if irodsclient_types.IsFileNotFoundError(err) {
var fileNotFoundError *irodsclient_types.FileNotFoundError
if errors.As(err, &fileNotFoundError) {
fmt.Fprintf(os.Stderr, "File or dir '%s' not found!\n", fileNotFoundError.Path)
commons.PrintErrorf("File or directory %q is not found!\n", fileNotFoundError.Path)
} else {
fmt.Fprintf(os.Stderr, "File or dir not found!\n")
commons.PrintErrorf("File or directory is not found!\n")
}
} else if irodsclient_types.IsCollectionNotEmptyError(err) {
var collectionNotEmptyError *irodsclient_types.CollectionNotEmptyError
if errors.As(err, &collectionNotEmptyError) {
fmt.Fprintf(os.Stderr, "Dir '%s' not empty!\n", collectionNotEmptyError.Path)
commons.PrintErrorf("Directory %q is not empty!\n", collectionNotEmptyError.Path)
} else {
fmt.Fprintf(os.Stderr, "Dir not empty!\n")
commons.PrintErrorf("Directory is not empty!\n")
}
} else if irodsclient_types.IsFileAlreadyExistError(err) {
var fileAlreadyExistError *irodsclient_types.FileAlreadyExistError
if errors.As(err, &fileAlreadyExistError) {
fmt.Fprintf(os.Stderr, "File or dir '%s' already exist!\n", fileAlreadyExistError.Path)
commons.PrintErrorf("File or directory %q already exists!\n", fileAlreadyExistError.Path)
} else {
fmt.Fprintf(os.Stderr, "File or dir already exist!\n")
commons.PrintErrorf("File or directory already exists!\n")
}
} else if irodsclient_types.IsTicketNotFoundError(err) {
var ticketNotFoundError *irodsclient_types.TicketNotFoundError
if errors.As(err, &ticketNotFoundError) {
fmt.Fprintf(os.Stderr, "Ticket '%s' not found!\n", ticketNotFoundError.Ticket)
commons.PrintErrorf("Ticket %q is not found!\n", ticketNotFoundError.Ticket)
} else {
fmt.Fprintf(os.Stderr, "Ticket not found!\n")
commons.PrintErrorf("Ticket is not found!\n")
}
} else if irodsclient_types.IsUserNotFoundError(err) {
var userNotFoundError *irodsclient_types.UserNotFoundError
if errors.As(err, &userNotFoundError) {
fmt.Fprintf(os.Stderr, "User '%s' not found!\n", userNotFoundError.Name)
commons.PrintErrorf("User %q is not found!\n", userNotFoundError.Name)
} else {
fmt.Fprintf(os.Stderr, "User not found!\n")
commons.PrintErrorf("User is not found!\n")
}
} else if irodsclient_types.IsIRODSError(err) {
var irodsError *irodsclient_types.IRODSError
if errors.As(err, &irodsError) {
fmt.Fprintf(os.Stderr, "iRODS Error (code: '%d', message: '%s')\n", irodsError.Code, irodsError.Error())
commons.PrintErrorf("iRODS Error (code: '%d', message: %q)\n", irodsError.Code, irodsError.Error())
} else {
commons.PrintErrorf("iRODS Error!\n")
}
} else if commons.IsNotDirError(err) {
var notDirError *commons.NotDirError
if errors.As(err, &notDirError) {
commons.PrintErrorf("Destination %q is not a directory!\n", notDirError.Path)
} else {
fmt.Fprintf(os.Stderr, "iRODS Error!\n")
commons.PrintErrorf("Destination is not a directory!\n")
}
} else {
commons.PrintErrorf("Unexpected error!\nError Trace:\n - %+v\n", err)
}

fmt.Fprintf(os.Stderr, "\nError Trace:\n - %+v\n", err)
os.Exit(1)
}
}
Loading

0 comments on commit 6bd9342

Please sign in to comment.