Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

single fileops use DoMultiOperation #506

Merged
merged 19 commits into from
Aug 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions cmd/copy.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"

"github.com/0chain/gosdk/constants"
"github.com/0chain/gosdk/zboxcore/sdk"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -40,9 +41,15 @@ var copyCmd = &cobra.Command{
remotePath := cmd.Flag("remotepath").Value.String()
destPath := cmd.Flag("destpath").Value.String()

err = allocationObj.CopyObject(remotePath, destPath)
err = allocationObj.DoMultiOperation([]sdk.OperationRequest{
{
OperationType: constants.FileOperationCopy,
RemotePath: remotePath,
DestPath: destPath,
},
})
if err != nil {
PrintError("Error performing CopyObject", err)
PrintError("Copy failed.", err)
os.Exit(1)
}

Expand Down
14 changes: 8 additions & 6 deletions cmd/createdir.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"

"github.com/0chain/gosdk/constants"
"github.com/0chain/gosdk/zboxcore/sdk"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -32,14 +33,15 @@ var createDirCmd = &cobra.Command{
}
dirname := cmd.Flag("dirname").Value.String()

if err != nil {
PrintError("CreateDir failed: ", err)
os.Exit(1)
}
err = allocationObj.CreateDir(dirname)
err = allocationObj.DoMultiOperation([]sdk.OperationRequest{
{
OperationType: constants.FileOperationCreateDir,
RemotePath: dirname,
},
})

if err != nil {
PrintError("CreateDir failed: ", err)
PrintError("Directory creation failed.", err)
os.Exit(1)
}

Expand Down
14 changes: 10 additions & 4 deletions cmd/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"

"github.com/0chain/gosdk/constants"
"github.com/0chain/gosdk/zboxcore/sdk"
"github.com/spf13/cobra"
)
Expand All @@ -15,12 +16,12 @@ var deleteCmd = &cobra.Command{
Long: `delete file from blobbers`,
Args: cobra.MinimumNArgs(0),
Run: func(cmd *cobra.Command, args []string) {
fflags := cmd.Flags() // fflags is a *flag.FlagSet
if fflags.Changed("allocation") == false { // check if the flag "path" is set
fflags := cmd.Flags() // fflags is a *flag.FlagSet
if !fflags.Changed("allocation") { // check if the flag "path" is set
PrintError("Error: allocation flag is missing") // If not, we'll let the user know
os.Exit(1) // and return
}
if fflags.Changed("remotepath") == false {
if !fflags.Changed("remotepath") {
PrintError("Error: remotepath flag is missing")
os.Exit(1)
}
Expand All @@ -33,7 +34,12 @@ var deleteCmd = &cobra.Command{
}
remotePath := cmd.Flag("remotepath").Value.String()

err = allocationObj.DeleteFile(remotePath)
err = allocationObj.DoMultiOperation([]sdk.OperationRequest{
{
OperationType: constants.FileOperationDelete,
RemotePath: remotePath,
},
})
if err != nil {
PrintError("Delete failed.", err.Error())
os.Exit(1)
Expand Down
4 changes: 2 additions & 2 deletions cmd/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ var downloadCmd = &cobra.Command{
}
}
} else if len(remotePath) > 0 {
if fflags.Changed("allocation") == false { // check if the flag "path" is set
if !fflags.Changed("allocation") { // check if the flag "path" is set
PrintError("Error: allocation flag is missing") // If not, we'll let the user know
os.Exit(1) // and return
}
Expand Down Expand Up @@ -168,7 +168,7 @@ var downloadCmd = &cobra.Command{
}
}
} else if len(multidownloadJSON) > 0 {
if fflags.Changed("allocation") == false { // check if the flag "path" is set
if !fflags.Changed("allocation") { // check if the flag "path" is set
PrintError("Error: allocation flag is missing") // If not, we'll let the user know
os.Exit(1) // and return
}
Expand Down
11 changes: 9 additions & 2 deletions cmd/move.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"

"github.com/0chain/gosdk/constants"
"github.com/0chain/gosdk/zboxcore/sdk"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -38,9 +39,15 @@ var moveCmd = &cobra.Command{
remotePath := cmd.Flag("remotepath").Value.String()
destPath := cmd.Flag("destpath").Value.String()

err = allocationObj.MoveObject(remotePath, destPath)
err = allocationObj.DoMultiOperation([]sdk.OperationRequest{
{
OperationType: constants.FileOperationMove,
RemotePath: remotePath,
DestPath: destPath,
},
})
if err != nil {
PrintError("Error performing CopyObject", err)
PrintError("Move failed.", err)
os.Exit(1)
}

Expand Down
19 changes: 13 additions & 6 deletions cmd/rename.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"

"github.com/0chain/gosdk/constants"
"github.com/0chain/gosdk/core/pathutil"
"github.com/0chain/gosdk/zboxcore/sdk"
"github.com/spf13/cobra"
Expand All @@ -16,17 +17,17 @@ var renameCmd = &cobra.Command{
Long: `rename an object on blobbers`,
Args: cobra.MinimumNArgs(0),
Run: func(cmd *cobra.Command, args []string) {
fflags := cmd.Flags() // fflags is a *flag.FlagSet
if fflags.Changed("allocation") == false { // check if the flag "path" is set
fflags := cmd.Flags() // fflags is a *flag.FlagSet
if !fflags.Changed("allocation") { // check if the flag "path" is set
PrintError("Error: allocation flag is missing") // If not, we'll let the user know
os.Exit(1) // and os.Exit(1)
}
if fflags.Changed("remotepath") == false {
if !fflags.Changed("remotepath") {
PrintError("Error: remotepath flag is missing")
os.Exit(1)
}

if fflags.Changed("destname") == false {
if !fflags.Changed("destname") {
PrintError("Error: destname flag is missing")
os.Exit(1)
}
Expand All @@ -44,9 +45,15 @@ var renameCmd = &cobra.Command{
return
}

err = allocationObj.RenameObject(remotePath, destName)
err = allocationObj.DoMultiOperation([]sdk.OperationRequest{
{
OperationType: constants.FileOperationRename,
RemotePath: remotePath,
DestName: destName,
},
})
if err != nil {
PrintError(err.Error())
PrintError("Rename failed.", err)
os.Exit(1)
}
fmt.Println(remotePath + " renamed")
Expand Down
16 changes: 3 additions & 13 deletions cmd/update.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ var updateCmd = &cobra.Command{
Args: cobra.MinimumNArgs(0),
Run: func(cmd *cobra.Command, args []string) {
fflags := cmd.Flags()
if fflags.Changed("allocation") == false {
if !fflags.Changed("allocation") {
PrintError("Error: allocation flag is missing")
os.Exit(1)
}
if fflags.Changed("remotepath") == false {
if !fflags.Changed("remotepath") {
PrintError("Error: remotepath flag is missing")
os.Exit(1)
}
Expand All @@ -47,17 +47,7 @@ var updateCmd = &cobra.Command{

wg := &sync.WaitGroup{}
statusBar := &StatusBar{wg: wg}
wg.Add(1)

err = startChunkedUpload(cmd, allocationObj, chunkedUploadArgs{
localPath: localPath,
remotePath: remotePath,
thumbnailPath: thumbnailPath,
encrypt: encrypt,
chunkNumber: updateChunkNumber,
isUpdate: true,
// isRepair: false,
}, statusBar)
err = singleUpload(allocationObj, localPath, remotePath, thumbnailPath, encrypt, false, true, updateChunkNumber, statusBar)

if err != nil {
PrintError("Update failed.", err)
Expand Down
64 changes: 42 additions & 22 deletions cmd/upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,21 +61,10 @@ var uploadCmd = &cobra.Command{
if multiuploadJSON != "" {
err = multiUpload(allocationObj, localPath, multiuploadJSON, statusBar)
} else {
wg.Add(1)
err = startChunkedUpload(cmd, allocationObj,
chunkedUploadArgs{
localPath: localPath,
thumbnailPath: thumbnailPath,
remotePath: remotePath,
encrypt: encrypt,
webStreaming: webStreaming,
chunkNumber: uploadChunkNumber,
// isUpdate: false,
// isRepair: false,
}, statusBar)
err = singleUpload(allocationObj, localPath, remotePath, thumbnailPath, encrypt, webStreaming, false, uploadChunkNumber, statusBar)
}
if err != nil {
PrintError("Upload failed.", err.Error())
PrintError("Upload failed.", err)
os.Exit(1)
}
wg.Wait()
Expand All @@ -97,7 +86,7 @@ type chunkedUploadArgs struct {
isRepair bool
}

func startChunkedUpload(cmd *cobra.Command, allocationObj *sdk.Allocation, args chunkedUploadArgs, statusBar sdk.StatusCallback) error {
func startChunkedUpload(allocationObj *sdk.Allocation, args chunkedUploadArgs, statusBar sdk.StatusCallback) error {
fileReader, err := os.Open(args.localPath)
if err != nil {
return err
Expand Down Expand Up @@ -148,13 +137,14 @@ func startChunkedUpload(cmd *cobra.Command, allocationObj *sdk.Allocation, args
}

type MultiUploadOption struct {
FilePath string `json:"filePath,omitempty"`
FileName string `json:"fileName,omitempty"`
RemotePath string `json:"remotePath,omitempty"`
ThumbnailPath string `json:"thumbnailPath,omitempty"`
Encrypt bool `json:"encrypt,omitempty"`
ChunkNumber int `json:"chunkNumber,omitempty"`
IsUpdate bool `json:"isUpdate,omitempty"`
FilePath string `json:"filePath,omitempty"`
FileName string `json:"fileName,omitempty"`
RemotePath string `json:"remotePath,omitempty"`
ThumbnailPath string `json:"thumbnailPath,omitempty"`
Encrypt bool `json:"encrypt,omitempty"`
ChunkNumber int `json:"chunkNumber,omitempty"`
IsUpdate bool `json:"isUpdate,omitempty"`
IsWebstreaming bool `json:"isWebstreaming,omitempty"`
}

func multiUpload(allocationObj *sdk.Allocation, workdir, jsonMultiUploadOptions string, statusBar *StatusBar) error {
Expand All @@ -173,6 +163,34 @@ func multiUpload(allocationObj *sdk.Allocation, workdir, jsonMultiUploadOptions
return err
}

return multiUploadWithOptions(allocationObj, workdir, options, statusBar)
}

func singleUpload(allocationObj *sdk.Allocation, localPath, remotePath, thumbnailPath string, encrypt, isWebstreaming, isUpdate bool, chunkNumber int, statusBar *StatusBar) error {
fullRemotePath, fileName, err := fullPathAndFileNameForUpload(localPath, remotePath)
if err != nil {
return err
}
remotePath = pathutil.Dir(fullRemotePath) + "/"
options := []MultiUploadOption{
{
FilePath: localPath,
FileName: fileName,
RemotePath: remotePath,
ThumbnailPath: thumbnailPath,
Encrypt: encrypt,
ChunkNumber: chunkNumber,
IsUpdate: isUpdate,
IsWebstreaming: isWebstreaming,
},
}

workdir := util.GetHomeDir()

return multiUploadWithOptions(allocationObj, workdir, options, statusBar)
}

func multiUploadWithOptions(allocationObj *sdk.Allocation, workdir string, options []MultiUploadOption, statusBar *StatusBar) error {
totalUploads := len(options)
filePaths := make([]string, totalUploads)
fileNames := make([]string, totalUploads)
Expand All @@ -181,6 +199,7 @@ func multiUpload(allocationObj *sdk.Allocation, workdir, jsonMultiUploadOptions
chunkNumbers := make([]int, totalUploads)
encrypts := make([]bool, totalUploads)
isUpdates := make([]bool, totalUploads)
isWebstreaming := make([]bool, totalUploads)
for idx, option := range options {
statusBar.wg.Add(1)
filePaths[idx] = option.FilePath
Expand All @@ -190,9 +209,10 @@ func multiUpload(allocationObj *sdk.Allocation, workdir, jsonMultiUploadOptions
chunkNumbers[idx] = option.ChunkNumber
encrypts[idx] = option.Encrypt
isUpdates[idx] = option.IsUpdate
isWebstreaming[idx] = option.IsWebstreaming
}

return allocationObj.StartMultiUpload(workdir, filePaths, fileNames, thumbnailPaths, encrypts, chunkNumbers, remotePaths, isUpdates, statusBar)
return allocationObj.StartMultiUpload(workdir, filePaths, fileNames, thumbnailPaths, encrypts, chunkNumbers, remotePaths, isUpdates, isWebstreaming, statusBar)
}

func init() {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.18

require (
github.com/0chain/errors v1.0.3
github.com/0chain/gosdk v1.8.18-0.20230823172043-ea887d53014a
github.com/0chain/gosdk v1.8.18-0.20230827195410-39deea65eca2
github.com/icza/bitio v1.1.0
github.com/olekukonko/tablewriter v0.0.5
github.com/spf13/cobra v1.6.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs
github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E=
github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM=
github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc=
github.com/0chain/gosdk v1.8.18-0.20230823172043-ea887d53014a h1:SObqSMyLF+RAP+gLVN02DelKeAYDYAf3wa8f9RxuVIs=
github.com/0chain/gosdk v1.8.18-0.20230823172043-ea887d53014a/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4=
github.com/0chain/gosdk v1.8.18-0.20230827195410-39deea65eca2 h1:zfDlOSfXsTBgCF4XC4x01Jo7brQRewppYkpAop+KM28=
github.com/0chain/gosdk v1.8.18-0.20230827195410-39deea65eca2/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Luzifer/go-openssl/v3 v3.1.0 h1:QqKqo6kYXGGUsvtUoCpRZm8lHw+jDfhbzr36gVj+/gw=
Expand Down
Loading