From c05cb20c5be6c57aa9e5d82db5ae1d44fab90407 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Sat, 19 Aug 2023 01:38:36 +0530 Subject: [PATCH 1/7] Timestamp changes (#1210) * minor changes to chalk in zus overview * check start time * parse form in parallel * fix err msg * don't parse for delete * valid sig fix * check form * fixed configs * rmv log * change pending check * use buf * rmv log --------- Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Co-authored-by: Harshit Mehndiratta <65766301+hm90121@users.noreply.github.com> Co-authored-by: dabasov --- README.md | 4 +--- .../blobbercore/allocation/allocationchange.go | 1 - .../0chain.net/blobbercore/allocation/entity.go | 15 ++++++++------- .../0chain.net/blobbercore/allocation/protocol.go | 1 + .../0chain.net/blobbercore/filestore/storage.go | 4 ++-- .../blobbercore/handler/file_command_upload.go | 1 - .../handler/object_operation_handler.go | 12 ++++++------ .../blobbercore/readmarker/readmarker.go | 5 ++++- .../blobbercore/writemarker/protocol.go | 5 ++++- code/go/0chain.net/core/transaction/entity.go | 1 + config/0chain_blobber.yaml | 2 +- goose/migrations/001_blobber_meta.sql | 3 ++- 12 files changed, 30 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 5ca2db433..03575ebf1 100644 --- a/README.md +++ b/README.md @@ -32,13 +32,11 @@ Users can also add their own servers to the network to operate in a hybrid cloud The [privacy protocol](https://zus.network/build) from Züs is unique where a user can easily share their encrypted data with their business partners, friends, and family through a proxy key sharing protocol, where the key is given to the providers, and they re-encrypt the data using the proxy key so that only the recipient can decrypt it with their private key. -Züs has ecosystem apps to encourage traditional storage consumption such as [Blimp](https://blimp.software/), a S3 server and cloud migration platform, and [Vult](https://vult.network/), a personal cloud app to store encrypted data and share privately with friends and family, and [Chalk](https://chalk.software/), a zero upfront cost permanent storage solution for NFT artists. +Züs has ecosystem apps to encourage traditional storage consumption such as [Blimp](https://blimp.software/), a S3 server and cloud migration platform, and [Vult](https://vult.network/), a personal cloud app to store encrypted data and share privately with friends and family, and [Chalk](https://chalk.software/), a high-performance story-telling storage solution for NFT artists. Other apps are [Bolt](https://bolt.holdings/), a wallet that is very secure with air-gapped 2FA split-key protocol to prevent hacks from compromising your digital assets, and it enables you to stake and earn from the storage providers; [Atlus](https://atlus.cloud/), a blockchain explorer and [Chimney](https://demo.chimney.software/), which allows anyone to join the network and earn using their server or by just renting one, with no prior knowledge required. - - ## Initial Setup ### Required OS and Software Dependencies diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index ba39c8a8c..4226c41f6 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -217,7 +217,6 @@ func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, allocatio return err } } - logging.Logger.Info("ApplyChanges", zap.Any("rootRef", rootRef)) _, err = rootRef.CalculateHash(ctx, true) return err } diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go index 4530a8322..4022e813f 100644 --- a/code/go/0chain.net/blobbercore/allocation/entity.go +++ b/code/go/0chain.net/blobbercore/allocation/entity.go @@ -48,13 +48,14 @@ type Allocation struct { RepairerID string `gorm:"column:repairer_id;size:64;not null"` Expiration common.Timestamp `gorm:"column:expiration_date;not null"` // AllocationRoot allcation_root of last write_marker - AllocationRoot string `gorm:"column:allocation_root;size:64;not null;default:''"` - FileMetaRoot string `gorm:"column:file_meta_root;size:64;not null;default:''"` - BlobberSize int64 `gorm:"column:blobber_size;not null;default:0"` - BlobberSizeUsed int64 `gorm:"column:blobber_size_used;not null;default:0"` - LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker;size:64"` - IsRedeemRequired bool `gorm:"column:is_redeem_required"` - TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"` + AllocationRoot string `gorm:"column:allocation_root;size:64;not null;default:''"` + FileMetaRoot string `gorm:"column:file_meta_root;size:64;not null;default:''"` + BlobberSize int64 `gorm:"column:blobber_size;not null;default:0"` + BlobberSizeUsed int64 `gorm:"column:blobber_size_used;not null;default:0"` + LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker;size:64"` + IsRedeemRequired bool `gorm:"column:is_redeem_required"` + TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"` + StartTime common.Timestamp `gorm:"column:start_time;not null"` // Ending and cleaning CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"` Finalized bool `gorm:"column:finalized;not null;default:false"` diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index 7edd76cfa..b38d0626b 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -108,6 +108,7 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc a.Finalized = sa.Finalized a.TimeUnit = sa.TimeUnit a.FileOptions = sa.FileOptions + a.StartTime = sa.StartTime m := map[string]interface{}{ "allocation_id": a.ID, diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 3d3864a84..56cc92979 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -83,8 +83,8 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i if err != nil { return nil, common.NewError("file_seek_error", err.Error()) } - - writtenSize, err := io.Copy(f, infile) + buf := make([]byte, BufferSize) + writtenSize, err := io.CopyBuffer(f, infile, buf) if err != nil { return nil, common.NewError("file_write_error", err.Error()) } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index f22c85948..101d5ef79 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -50,7 +50,6 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request } fileChanger := &allocation.UploadFileChanger{} - uploadMetaString := req.FormValue(UploadMeta) err := json.Unmarshal([]byte(uploadMetaString), fileChanger) if err != nil { diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 7efc481be..5eea26c2b 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -494,12 +494,6 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b return nil, common.NewError("invalid_parameters", "Invalid connection id passed") } - err = checkPendingMarkers(ctx, allocationObj.ID) - if err != nil { - Logger.Error("Error checking pending markers", zap.Error(err)) - return nil, common.NewError("pending_markers", "previous marker is still pending to be redeemed") - } - // Lock will compete with other CommitWrites and Challenge validation mutex := lock.GetMutex(allocationObj.TableName(), allocationID) mutex.Lock() @@ -507,6 +501,12 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b elapsedGetLock := time.Since(startTime) - elapsedAllocation + err = checkPendingMarkers(ctx, allocationObj.ID) + if err != nil { + Logger.Error("Error checking pending markers", zap.Error(err)) + return nil, common.NewError("pending_markers", "previous marker is still pending to be redeemed") + } + connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) if err != nil { diff --git a/code/go/0chain.net/blobbercore/readmarker/readmarker.go b/code/go/0chain.net/blobbercore/readmarker/readmarker.go index 7d90819ef..4522f7129 100644 --- a/code/go/0chain.net/blobbercore/readmarker/readmarker.go +++ b/code/go/0chain.net/blobbercore/readmarker/readmarker.go @@ -103,8 +103,11 @@ func (rm *ReadMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.All return common.NewError("read_marker_validation_failed", "Read Marker clientID does not match request clientID") } + if rm.LatestRM.Timestamp < sa.StartTime { + return common.NewError("read_marker_validation_failed", "Readmarker timestamp is before the allocation start time") + } + if rm.LatestRM.Timestamp > sa.Expiration { - zLogger.Logger.Error("Readmarker is for an expired allocation", zap.Any("rm", rm)) return common.NewError("read_marker_validation_failed", "Readmarker is for an expired allocation") } diff --git a/code/go/0chain.net/blobbercore/writemarker/protocol.go b/code/go/0chain.net/blobbercore/writemarker/protocol.go index bca1b19e2..cfe950b40 100644 --- a/code/go/0chain.net/blobbercore/writemarker/protocol.go +++ b/code/go/0chain.net/blobbercore/writemarker/protocol.go @@ -81,10 +81,13 @@ func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *al if clientID == "" || clientID != wme.WM.ClientID || clientID != co.ClientID || co.ClientID != wme.WM.ClientID { return common.NewError("write_marker_validation_failed", "Write Marker is not by the same client who uploaded") } + if wme.WM.Timestamp < dbAllocation.StartTime { + return common.NewError("write_marker_validation_failed", "Write Marker timestamp is before the allocation start time") + } currTime := common.Now() // blobber clock is allowed to be 10 seconds behind the current time - if wme.WM.Timestamp > currTime+10 { + if wme.WM.Timestamp > currTime+60 { return common.NewError("write_marker_validation_failed", "Write Marker timestamp is in the future") } diff --git a/code/go/0chain.net/core/transaction/entity.go b/code/go/0chain.net/core/transaction/entity.go index 43fc030f5..3d096d5a0 100644 --- a/code/go/0chain.net/core/transaction/entity.go +++ b/code/go/0chain.net/core/transaction/entity.go @@ -90,6 +90,7 @@ type StorageAllocation struct { TimeUnit time.Duration `json:"time_unit"` WritePool uint64 `json:"write_pool"` FileOptions uint16 `json:"file_options"` + StartTime common.Timestamp `json:"start_time"` DataShards int64 `json:"data_shards"` ParityShards int64 `json:"parity_shards"` diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml index 8fad49b25..b4c845b25 100755 --- a/config/0chain_blobber.yaml +++ b/config/0chain_blobber.yaml @@ -24,7 +24,7 @@ max_dirs_files: 50000 # delegate wallet (must be set) delegate_wallet: '9c693cb14f29917968d6e8c909ebbea3425b4c1bc64b6732cadc2a1869f49be9' -`# maximum allowed number of stake holders +# maximum allowed number of stake holders num_delegates: 50 # service charge of the blobber service_charge: 0.10 diff --git a/goose/migrations/001_blobber_meta.sql b/goose/migrations/001_blobber_meta.sql index e4d2158ba..86b3883d0 100644 --- a/goose/migrations/001_blobber_meta.sql +++ b/goose/migrations/001_blobber_meta.sql @@ -89,7 +89,8 @@ CREATE TABLE public.allocations ( time_unit bigint DEFAULT '172800000000000'::bigint NOT NULL, cleaned_up boolean DEFAULT false NOT NULL, finalized boolean DEFAULT false NOT NULL, - file_options integer DEFAULT 63 NOT NULL + file_options integer DEFAULT 63 NOT NULL, + start_time bigint NOT NULL ); From 0a018708f9b3b00423a273c8dae42c981349a5c9 Mon Sep 17 00:00:00 2001 From: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> Date: Sun, 20 Aug 2023 02:13:32 +0530 Subject: [PATCH 2/7] added timeout if build is stuck (#1218) * added timeout if build is stuck * increased timeout for blobber build --- .github/workflows/build-&-publish-docker-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index e6322feec..a91093587 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -21,6 +21,7 @@ env: jobs: blobber: + timeout-minutes: 25 runs-on: [self-hosted, arc-runner] steps: - name: Set docker image tag @@ -95,6 +96,7 @@ jobs: ./docker.local/bin/build.blobber.sh validator: + timeout-minutes: 20 runs-on: [self-hosted, arc-runner] steps: - name: Set docker image tag From 090e715c29d7451f9c4231be8e49243e64e61b09 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Wed, 23 Aug 2023 15:20:13 +0530 Subject: [PATCH 3/7] Calculate hash on upload (#1217) * minor changes to chalk in zus overview * check start time * parse form in parallel * fix err msg * don't parse for delete * valid sig fix * check form * fixed configs * rmv log * change pending check * use buf * rmv log * add vt and mt calc on fly * fix size calculation * check hasher * copy from file * allow read * fix thumbnail write * fix unit test * init hasher * fix alloc unit test * check alloc root --------- Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Co-authored-by: Harshit Mehndiratta <65766301+hm90121@users.noreply.github.com> Co-authored-by: dabasov --- .../blobbercore/allocation/connection.go | 42 +++++++++++++++++-- .../allocation/file_changer_base.go | 5 +++ .../allocation/file_changer_upload_test.go | 5 +++ .../allocation/updatefilechange_test.go | 5 +++ .../blobbercore/filestore/storage.go | 35 +++++++--------- .../0chain.net/blobbercore/filestore/store.go | 6 ++- .../blobbercore/filestore/store_test.go | 31 ++++++++++---- .../blobbercore/filestore/tree_validation.go | 18 ++++---- .../handler/file_command_update.go | 30 ++++++++++++- .../handler/file_command_upload.go | 32 ++++++++++++-- .../blobbercore/handler/handler_test.go | 24 ++++++----- .../blobbercore/writemarker/protocol.go | 10 ++++- 12 files changed, 187 insertions(+), 56 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go index 612d7b141..ae366b768 100644 --- a/code/go/0chain.net/blobbercore/allocation/connection.go +++ b/code/go/0chain.net/blobbercore/allocation/connection.go @@ -4,6 +4,8 @@ import ( "context" "sync" "time" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" ) var ( @@ -15,18 +17,23 @@ var ( var ( connectionObjSizeMap = make(map[string]*ConnectionObjSize) - connectionObjMutex sync.Mutex + connectionObjMutex sync.RWMutex ) type ConnectionObjSize struct { Size int64 UpdatedAt time.Time + Changes map[string]*ConnectionChanges +} + +type ConnectionChanges struct { + Hasher *filestore.CommitHasher } // GetConnectionObjSize gets the connection size from the memory func GetConnectionObjSize(connectionID string) int64 { - connectionObjMutex.Lock() - defer connectionObjMutex.Unlock() + connectionObjMutex.RLock() + defer connectionObjMutex.RUnlock() connectionObjSize := connectionObjSizeMap[connectionID] if connectionObjSize == nil { return 0 @@ -43,6 +50,7 @@ func UpdateConnectionObjSize(connectionID string, addSize int64) { connectionObjSizeMap[connectionID] = &ConnectionObjSize{ Size: addSize, UpdatedAt: time.Now(), + Changes: make(map[string]*ConnectionChanges), } return } @@ -51,6 +59,34 @@ func UpdateConnectionObjSize(connectionID string, addSize int64) { connectionObjSize.UpdatedAt = time.Now() } +func GetHasher(connectionID, pathHash string) *filestore.CommitHasher { + connectionObjMutex.RLock() + defer connectionObjMutex.RUnlock() + connectionObj := connectionObjSizeMap[connectionID] + if connectionObj == nil { + return nil + } + if connectionObj.Changes[pathHash] == nil { + return nil + } + return connectionObj.Changes[pathHash].Hasher +} + +func UpdateConnectionObjWithHasher(connectionID, pathHash string, hasher *filestore.CommitHasher) { + connectionObjMutex.Lock() + defer connectionObjMutex.Unlock() + connectionObj := connectionObjSizeMap[connectionID] + if connectionObj == nil { + connectionObjSizeMap[connectionID] = &ConnectionObjSize{ + UpdatedAt: time.Now(), + Changes: make(map[string]*ConnectionChanges), + } + } + connectionObjSizeMap[connectionID].Changes[pathHash] = &ConnectionChanges{ + Hasher: hasher, + } +} + // DeleteConnectionObjEntry remove the connectionID entry from map // If the given connectionID is not present, then it is no-op. func DeleteConnectionObjEntry(connectionID string) { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index 698b59974..c8e733dde 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -5,6 +5,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" ) // BaseFileChanger base file change processor @@ -92,6 +93,10 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context) error { fileInputData.ValidationRoot = fc.ValidationRoot fileInputData.FixedMerkleRoot = fc.FixedMerkleRoot fileInputData.ChunkSize = fc.ChunkSize + fileInputData.Hasher = GetHasher(fc.ConnectionID, encryption.Hash(fc.Path)) + if fileInputData.Hasher == nil { + return common.NewError("invalid_parameters", "Invalid parameters. Error getting hasher for commit.") + } _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData) if err != nil { return common.NewError("file_store_error", "Error committing to file store. "+err.Error()) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go index f75c8624f..6ee435013 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go @@ -14,6 +14,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/core/zcncrypto" @@ -98,6 +99,9 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) { ctx := datastore.GetStore().CreateTransaction(context.TODO()) fPath := "/new" + hasher := filestore.GetNewCommitHasher(2310) + pathHash := encryption.Hash(fPath) + UpdateConnectionObjWithHasher("connection_id", pathHash, hasher) change := &UploadFileChanger{ BaseFileChanger: BaseFileChanger{ Filename: filepath.Base(fPath), @@ -107,6 +111,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) { ValidationRoot: tc.validationRoot, Size: 2310, ChunkSize: 65536, + ConnectionID: "connection_id", }, } diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go index d6e2238e6..4452299ef 100644 --- a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go @@ -9,6 +9,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/core/zcncrypto" "github.com/0chain/gosdk/zboxcore/client" @@ -337,6 +338,9 @@ func TestBlobberCore_UpdateFile(t *testing.T) { tc.setupDbMock() ctx := datastore.GetStore().CreateTransaction(context.TODO()) + hasher := filestore.GetNewCommitHasher(2310) + pathHash := encryption.Hash(tc.path) + UpdateConnectionObjWithHasher("connection_id", pathHash, hasher) change := &UpdateFileChanger{ BaseFileChanger: BaseFileChanger{ @@ -352,6 +356,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) { ThumbnailSize: 92, ChunkSize: 65536, IsFinal: true, + ConnectionID: "connection_id", }, } diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 56cc92979..0a1781d13 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -59,7 +59,7 @@ const ( ) func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, infile multipart.File) (*FileOutputData, error) { - tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, encryption.Hash(fileData.Path), conID) + tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, fileData.FilePathHash, conID) var initialSize int64 finfo, err := os.Stat(tempFilePath) if err != nil && !errors.Is(err, os.ErrNotExist) { @@ -73,7 +73,7 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i return nil, common.NewError("dir_creation_error", err.Error()) } - f, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_WRONLY, 0644) + f, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_RDWR, 0644) if err != nil { return nil, common.NewError("file_open_error", err.Error()) } @@ -88,6 +88,17 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i if err != nil { return nil, common.NewError("file_write_error", err.Error()) } + if !fileData.IsThumbnail { + _, err = f.Seek(fileData.UploadOffset, io.SeekStart) + if err != nil { + return nil, common.NewError("file_seek_error", err.Error()) + } + + _, err = io.CopyBuffer(fileData.Hasher, f, buf) + if err != nil { + return nil, common.NewError("file_read_error", err.Error()) + } + } finfo, err = f.Stat() if err != nil { @@ -243,27 +254,18 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) } fileSize := rStat.Size() - hasher := GetNewCommitHasher(fileSize) bufSize := BufferSize if fileSize < BufferSize { bufSize = int(fileSize) } buffer := make([]byte, bufSize) - _, err = io.CopyBuffer(hasher, r, buffer) - if err != nil { - return false, common.NewError("read_write_error", err.Error()) - } - err = hasher.Finalize() - if err != nil { - return false, common.NewError("finalize_error", err.Error()) - } - fmtRootBytes, err := hasher.fmt.CalculateRootAndStoreNodes(f) + fmtRootBytes, err := fileData.Hasher.fmt.CalculateRootAndStoreNodes(f) if err != nil { return false, common.NewError("fmt_hash_calculation_error", err.Error()) } - validationRootBytes, err := hasher.vt.CalculateRootAndStoreNodes(f) + validationRootBytes, err := fileData.Hasher.vt.CalculateRootAndStoreNodes(f) if err != nil { return false, common.NewError("validation_hash_calculation_error", err.Error()) } @@ -279,12 +281,7 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) "calculated validation root does not match with client's validation root") } - _, err = r.Seek(0, io.SeekStart) - if err != nil { - return false, common.NewError("seek_error", err.Error()) - } - - _, err = io.Copy(f, r) + _, err = io.CopyBuffer(f, r, buffer) if err != nil { return false, common.NewError("write_error", err.Error()) } diff --git a/code/go/0chain.net/blobbercore/filestore/store.go b/code/go/0chain.net/blobbercore/filestore/store.go index 481395fe8..9d4a614e7 100644 --- a/code/go/0chain.net/blobbercore/filestore/store.go +++ b/code/go/0chain.net/blobbercore/filestore/store.go @@ -20,8 +20,10 @@ type FileInputData struct { //Upload-Offset indicates a byte offset within a resource. The value MUST be a non-negative integer. UploadOffset int64 //IsFinal the request is final chunk - IsFinal bool - IsThumbnail bool + IsFinal bool + IsThumbnail bool + FilePathHash string + Hasher *CommitHasher } type FileOutputData struct { diff --git a/code/go/0chain.net/blobbercore/filestore/store_test.go b/code/go/0chain.net/blobbercore/filestore/store_test.go index 340da9581..bcef5f2ec 100644 --- a/code/go/0chain.net/blobbercore/filestore/store_test.go +++ b/code/go/0chain.net/blobbercore/filestore/store_test.go @@ -248,23 +248,26 @@ func TestStoreStorageWriteAndCommit(t *testing.T) { size := 640 * KB validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size)) require.Nil(t, err) - + pathHash := encryption.Hash(test.remotePath) + hasher := GetNewCommitHasher(int64(size)) fid := &FileInputData{ Name: test.fileName, Path: test.remotePath, ValidationRoot: validationRoot, FixedMerkleRoot: fixedMerkleRoot, ChunkSize: 64 * KB, + FilePathHash: pathHash, + Hasher: hasher, } f, err := os.Open(fPath) require.Nil(t, err) defer f.Close() - _, err = fs.WriteFile(test.allocID, test.connID, fid, f) require.Nil(t, err) + err = hasher.Finalize() + require.Nil(t, err) - pathHash := encryption.Hash(test.remotePath) tempFilePath := fs.getTempPathForFile(test.allocID, test.fileName, pathHash, test.connID) tF, err := os.Stat(tempFilePath) require.Nil(t, err) @@ -329,13 +332,16 @@ func TestDeletePreCommitDir(t *testing.T) { size := 640 * KB validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size)) require.Nil(t, err) - + pathHash := encryption.Hash(remotePath) + hasher := GetNewCommitHasher(int64(size)) fid := &FileInputData{ Name: fileName, Path: remotePath, ValidationRoot: validationRoot, FixedMerkleRoot: fixedMerkleRoot, ChunkSize: 64 * KB, + FilePathHash: pathHash, + Hasher: hasher, } // checkc if file to be uploaded exists f, err := os.Open(fPath) @@ -344,9 +350,9 @@ func TestDeletePreCommitDir(t *testing.T) { _, err = fs.WriteFile(allocID, connID, fid, f) require.Nil(t, err) f.Close() - + err = hasher.Finalize() + require.Nil(t, err) // check if file is written to temp location - pathHash := encryption.Hash(remotePath) tempFilePath := fs.getTempPathForFile(allocID, fileName, pathHash, connID) tF, err := os.Stat(tempFilePath) require.Nil(t, err) @@ -368,6 +374,8 @@ func TestDeletePreCommitDir(t *testing.T) { fid.ValidationRoot = validationRoot fid.FixedMerkleRoot = fixedMerkleRoot + hasher = GetNewCommitHasher(int64(size)) + fid.Hasher = hasher // Write file to temp location f, err = os.Open(fPath) @@ -379,6 +387,8 @@ func TestDeletePreCommitDir(t *testing.T) { tempFilePath = fs.getTempPathForFile(allocID, fileName, pathHash, connID) _, err = os.Stat(tempFilePath) require.Nil(t, err) + err = hasher.Finalize() + require.Nil(t, err) success, err = fs.CommitWrite(allocID, connID, fid) require.Nil(t, err) @@ -422,13 +432,16 @@ func TestStorageUploadUpdate(t *testing.T) { size := 640 * KB validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size)) require.Nil(t, err) - + pathHash := encryption.Hash(remotePath) + hasher := GetNewCommitHasher(int64(size)) fid := &FileInputData{ Name: fileName, Path: remotePath, ValidationRoot: validationRoot, FixedMerkleRoot: fixedMerkleRoot, ChunkSize: 64 * KB, + FilePathHash: pathHash, + Hasher: hasher, } // checkc if file to be uploaded exists f, err := os.Open(fPath) @@ -437,9 +450,9 @@ func TestStorageUploadUpdate(t *testing.T) { _, err = fs.WriteFile(allocID, connID, fid, f) require.Nil(t, err) f.Close() - + err = hasher.Finalize() + require.Nil(t, err) // check if file is written to temp location - pathHash := encryption.Hash(remotePath) tempFilePath := fs.getTempPathForFile(allocID, fileName, pathHash, connID) tF, err := os.Stat(tempFilePath) require.Nil(t, err) diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation.go b/code/go/0chain.net/blobbercore/filestore/tree_validation.go index 33ae0eafe..b44145707 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation.go @@ -400,21 +400,25 @@ func getNewValidationTree(dataSize int64) *validationTree { // commitHasher is used to calculate and store tree nodes for fixed merkle tree and // validation tree when client commits file with the writemarker. -type commitHasher struct { +type CommitHasher struct { fmt *fixedMerkleTree vt *validationTree isInitialized bool } -func GetNewCommitHasher(dataSize int64) *commitHasher { - c := new(commitHasher) +func GetNewCommitHasher(dataSize int64) *CommitHasher { + c := new(CommitHasher) c.fmt = getNewFixedMerkleTree() c.vt = getNewValidationTree(dataSize) c.isInitialized = true return c } -func (c *commitHasher) Write(b []byte) (int, error) { +func (c *CommitHasher) Write(b []byte) (int, error) { + if !c.isInitialized || c.fmt == nil || c.vt == nil { + return 0, errors.New("commit hasher is not initialized") + } + var ( wg sync.WaitGroup errChan = make(chan error, 2) @@ -445,7 +449,7 @@ func (c *commitHasher) Write(b []byte) (int, error) { return n, nil } -func (c *commitHasher) Finalize() error { +func (c *CommitHasher) Finalize() error { var ( wg sync.WaitGroup errChan = make(chan error, 2) @@ -473,10 +477,10 @@ func (c *commitHasher) Finalize() error { return nil } -func (c *commitHasher) GetFixedMerkleRoot() string { +func (c *CommitHasher) GetFixedMerkleRoot() string { return c.fmt.GetMerkleRoot() } -func (c *commitHasher) GetValidationMerkleRoot() string { +func (c *CommitHasher) GetValidationMerkleRoot() string { return hex.EncodeToString(c.vt.GetValidationRoot()) } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index fd8b7c1d8..e62e2db4e 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -11,6 +11,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" sdkConst "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/zboxcore/fileref" @@ -94,17 +95,42 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ cmd.reloadChange(connectionObj) + if cmd.fileChanger.Size == 0 { + return result, common.NewError("invalid_parameters", "Invalid parameters. Size cannot be zero") + } + + var hasher *filestore.CommitHasher + filePathHash := encryption.Hash(cmd.fileChanger.Path) + if cmd.fileChanger.UploadOffset == 0 { + hasher = filestore.GetNewCommitHasher(cmd.fileChanger.Size) + allocation.UpdateConnectionObjWithHasher(connectionObj.ID, filePathHash, hasher) + } else { + hasher = allocation.GetHasher(connectionObj.ID, filePathHash) + if hasher == nil { + return result, common.NewError("invalid_parameters", "Invalid parameters. Error getting hasher for upload.") + } + } + fileInputData := &filestore.FileInputData{ Name: cmd.fileChanger.Filename, Path: cmd.fileChanger.Path, UploadOffset: cmd.fileChanger.UploadOffset, IsFinal: cmd.fileChanger.IsFinal, + FilePathHash: filePathHash, + Hasher: hasher, } fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionObj.ID, fileInputData, origfile) if err != nil { return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) } + if cmd.fileChanger.IsFinal { + err = hasher.Finalize() + if err != nil { + return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) + } + } + result.ValidationRoot = fileOutputData.ValidationRoot result.FixedMerkleRoot = fileOutputData.FixedMerkleRoot result.Size = fileOutputData.Size @@ -121,7 +147,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ } cmd.fileChanger.AllocationID = allocationObj.ID - cmd.fileChanger.Size += fileOutputData.Size + // cmd.fileChanger.Size += fileOutputData.Size cmd.allocationChange = &allocation.AllocationChange{} cmd.allocationChange.ConnectionID = connectionObj.ID @@ -145,7 +171,7 @@ func (cmd *UpdateFileCommand) ProcessThumbnail(ctx context.Context, req *http.Re if thumbHeader != nil { defer thumbfile.Close() - thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path} + thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: encryption.Hash(cmd.fileChanger.Path)} thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionObj.ID, thumbInputData, thumbfile) if err != nil { return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index 101d5ef79..b639bc6c2 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -13,6 +13,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/zboxcore/fileref" @@ -110,6 +111,22 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, req *http.Requ cmd.reloadChange(connectionObj) + var hasher *filestore.CommitHasher + filePathHash := encryption.Hash(cmd.fileChanger.Path) + if cmd.fileChanger.Size == 0 { + return result, common.NewError("invalid_parameters", "Invalid parameters. Size cannot be zero") + } + fmt.Println("cmd.fileChanger.Size", cmd.fileChanger.Size) + if cmd.fileChanger.UploadOffset == 0 { + hasher = filestore.GetNewCommitHasher(cmd.fileChanger.Size) + allocation.UpdateConnectionObjWithHasher(connectionObj.ID, filePathHash, hasher) + } else { + hasher = allocation.GetHasher(connectionObj.ID, filePathHash) + if hasher == nil { + return result, common.NewError("invalid_parameters", "Error getting hasher for upload.") + } + } + fileInputData := &filestore.FileInputData{ Name: cmd.fileChanger.Filename, Path: cmd.fileChanger.Path, @@ -117,10 +134,19 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, req *http.Requ ChunkSize: cmd.fileChanger.ChunkSize, UploadOffset: cmd.fileChanger.UploadOffset, IsFinal: cmd.fileChanger.IsFinal, + FilePathHash: filePathHash, + Hasher: hasher, } fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionObj.ID, fileInputData, origfile) if err != nil { - return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) + return result, common.NewError("upload_error", "Failed to write file. "+err.Error()) + } + + if cmd.fileChanger.IsFinal { + err = hasher.Finalize() + if err != nil { + return result, common.NewError("upload_error", "Failed to finalize the hasher. "+err.Error()) + } } result.Filename = cmd.fileChanger.Filename @@ -140,7 +166,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, req *http.Requ } cmd.fileChanger.AllocationID = allocationObj.ID - cmd.fileChanger.Size += fileOutputData.Size + // cmd.fileChanger.Size += fileOutputData.Size cmd.allocationChange = &allocation.AllocationChange{} cmd.allocationChange.ConnectionID = connectionObj.ID @@ -159,7 +185,7 @@ func (cmd *UploadFileCommand) ProcessThumbnail(ctx context.Context, req *http.Re if thumbHeader != nil { defer thumbfile.Close() - thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path} + thumbInputData := &filestore.FileInputData{Name: thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: encryption.Hash(cmd.fileChanger.Path)} thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionObj.ID, thumbInputData, thumbfile) if err != nil { return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 69f7de720..1c7343e20 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -756,11 +756,20 @@ func TestHandlers_Requiring_Signature(t *testing.T) { if err != nil { t.Fatal() } - + root, _ := os.Getwd() + file, err := os.Open(root + "/handler_test.go") + if err != nil { + t.Fatal(err) + } + stat, err := file.Stat() + if err != nil { + t.Fatal(err) + } + size := stat.Size() q := url.Query() formFieldByt, err := json.Marshal( &allocation.UploadFileChanger{ - BaseFileChanger: allocation.BaseFileChanger{Path: path}}) + BaseFileChanger: allocation.BaseFileChanger{Path: path, Size: size}}) if err != nil { t.Fatal(err) } @@ -772,20 +781,15 @@ func TestHandlers_Requiring_Signature(t *testing.T) { body := bytes.NewBuffer(nil) formWriter := multipart.NewWriter(body) - root, _ := os.Getwd() - file, err := os.Open(root + "/handler_test.go") - if err != nil { - t.Fatal(err) - } fileField, err := formWriter.CreateFormFile("uploadFile", file.Name()) if err != nil { t.Fatal(err) } - fileB := make([]byte, 0) - if _, err := io.ReadFull(file, fileB); err != nil { + data, err := io.ReadAll(file) + if err != nil { t.Fatal(err) } - if _, err := fileField.Write(fileB); err != nil { + if _, err := fileField.Write(data); err != nil { t.Fatal(err) } if err := formWriter.Close(); err != nil { diff --git a/code/go/0chain.net/blobbercore/writemarker/protocol.go b/code/go/0chain.net/blobbercore/writemarker/protocol.go index cfe950b40..b0745aae0 100644 --- a/code/go/0chain.net/blobbercore/writemarker/protocol.go +++ b/code/go/0chain.net/blobbercore/writemarker/protocol.go @@ -57,6 +57,10 @@ func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *al return common.NewError("write_marker_validation_failed", "Signature exceeds maximum length") } + if wme.WM.AllocationRoot == dbAllocation.AllocationRoot { + return common.NewError("write_marker_validation_failed", "Write Marker allocation root is the same as the allocation root on record") + } + if wme.WM.PreviousAllocationRoot != dbAllocation.AllocationRoot { return common.NewError("invalid_write_marker", "Invalid write marker. Prev Allocation root does not match the allocation root on record") } @@ -86,7 +90,7 @@ func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *al } currTime := common.Now() - // blobber clock is allowed to be 10 seconds behind the current time + // blobber clock is allowed to be 60 seconds behind the current time if wme.WM.Timestamp > currTime+60 { return common.NewError("write_marker_validation_failed", "Write Marker timestamp is in the future") } @@ -185,6 +189,10 @@ func (wme *WriteMarkerEntity) VerifyRollbackMarker(ctx context.Context, dbAlloca return common.NewError("empty write_marker_validation_failed", fmt.Sprintf("Write Marker size is %v but should be 0", wme.WM.Size)) } + if wme.WM.AllocationRoot == dbAllocation.AllocationRoot { + return common.NewError("write_marker_validation_failed", "Write Marker allocation root is the same as the allocation root on record") + } + if wme.WM.AllocationRoot != latestWM.WM.PreviousAllocationRoot { return common.NewError("write_marker_validation_failed", fmt.Sprintf("Write Marker allocation root %v does not match the previous allocation root of latest write marker %v", wme.WM.AllocationRoot, latestWM.WM.PreviousAllocationRoot)) } From e154a63b60366ae09ee013037bf0f03fa1781535 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Thu, 24 Aug 2023 02:36:18 +0530 Subject: [PATCH 4/7] Fix pre download (#1222) * fix pre download * rmv hash --- .../0chain.net/blobbercore/allocation/file_changer_base.go | 5 +++-- code/go/0chain.net/blobbercore/reference/ref.go | 2 ++ goose/migrations/001_blobber_meta.sql | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index c8e733dde..ca67a9f79 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -45,8 +45,9 @@ type BaseFileChanger struct { ThumbnailSize int64 `json:"thumbnail_size"` ThumbnailFilename string `json:"thumbnail_filename"` - EncryptedKey string `json:"encrypted_key,omitempty"` - CustomMeta string `json:"custom_meta,omitempty"` + EncryptedKey string `json:"encrypted_key,omitempty"` + EncryptedKeyPoint string `json:"encrypted_key_point,omitempty"` + CustomMeta string `json:"custom_meta,omitempty"` ChunkSize int64 `json:"chunk_size,omitempty"` // the size of achunk. 64*1024 is default IsFinal bool `json:"is_final,omitempty"` // current chunk is last or not diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 2f26103e6..5b8d42d5d 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -59,6 +59,7 @@ type Ref struct { ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size;not null;default:0" filelist:"actual_thumbnail_size"` ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash;size:64;not null" filelist:"actual_thumbnail_hash"` EncryptedKey string `gorm:"column:encrypted_key;size:64" filelist:"encrypted_key"` + EncryptedKeyPoint string `gorm:"column:encrypted_key_point;size:64" filelist:"encrypted_key_point"` Children []*Ref `gorm:"-"` childrenLoaded bool CreatedAt common.Timestamp `gorm:"column:created_at;index:idx_created_at,sort:desc" dirlist:"created_at" filelist:"created_at"` @@ -118,6 +119,7 @@ type PaginatedRef struct { //Gorm smart select fields. ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size" json:"actual_thumbnail_size,omitempty"` ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash" json:"actual_thumbnail_hash,omitempty"` EncryptedKey string `gorm:"column:encrypted_key" json:"encrypted_key,omitempty"` + EncryptedKeyPoint string `gorm:"column:encrypted_key_point" json:"encrypted_key_point,omitempty"` CreatedAt common.Timestamp `gorm:"column:created_at" json:"created_at,omitempty"` UpdatedAt common.Timestamp `gorm:"column:updated_at" json:"updated_at,omitempty"` diff --git a/goose/migrations/001_blobber_meta.sql b/goose/migrations/001_blobber_meta.sql index 86b3883d0..470430e04 100644 --- a/goose/migrations/001_blobber_meta.sql +++ b/goose/migrations/001_blobber_meta.sql @@ -332,6 +332,7 @@ CREATE TABLE public.reference_objects ( actual_thumbnail_size bigint DEFAULT 0 NOT NULL, actual_thumbnail_hash character varying(64) NOT NULL, encrypted_key character varying(64), + encrypted_key_point character varying(64), created_at bigint, updated_at bigint, deleted_at timestamp with time zone, From b7df47b8526297cf096c3025dae4a118cad57fe0 Mon Sep 17 00:00:00 2001 From: Dinmukhammed Kambarov <52813950+din-mukhammed@users.noreply.github.com> Date: Mon, 28 Aug 2023 20:44:38 +0600 Subject: [PATCH 5/7] Fix forbid_rename option from denying uploads (#1214) * minor changes to chalk in zus overview * fix forbid_rename from denying uploads * fixed configs * remove redundant checks --------- Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Co-authored-by: Harshit Mehndiratta <65766301+hm90121@users.noreply.github.com> Co-authored-by: dabasov --- .../handler/file_command_upload.go | 5 ----- .../handler/object_operation_handler.go | 20 ++++++++----------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index b639bc6c2..d83e6b010 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -78,11 +78,6 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("duplicate_file", msg) } - if allocationObj.OwnerID != clientID && - allocationObj.RepairerID != clientID { - return common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") - } - _, thumbHeader, _ := req.FormFile(UploadThumbnailFile) if thumbHeader != nil { if thumbHeader.Size > MaxThumbnailSize { diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 5eea26c2b..913530166 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -427,22 +427,22 @@ func (fsh *StorageHandler) CreateConnection(ctx context.Context, r *http.Request return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } - if !allocationObj.CanRename() { - return nil, common.NewError("prohibited_allocation_file_options", "Cannot rename data in this allocation.") - } - clientID := ctx.Value(constants.ContextKeyClient).(string) _ = ctx.Value(constants.ContextKeyClientKey).(string) + if clientID == "" { + return nil, common.NewError("invalid_operation", "Invalid client") + } + + if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID { + return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") + } + valid, err := verifySignatureFromRequest(allocationTx, r.Header.Get(common.ClientSignatureHeader), allocationObj.OwnerPublicKey) if !valid || err != nil { return nil, common.NewError("invalid_signature", "Invalid signature") } - if clientID == "" { - return nil, common.NewError("invalid_operation", "Invalid client") - } - connectionID := r.FormValue("connection_id") if connectionID == "" { return nil, common.NewError("invalid_parameters", "Invalid connection id passed") @@ -1199,10 +1199,6 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*blo return nil, common.NewError("invalid_signature", "Invalid signature") } - if clientID == "" { - return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") - } - connectionID, ok := common.GetField(r, "connection_id") if !ok { return nil, common.NewError("invalid_parameters", "Invalid connection id passed") From 248367c18a4affe65e34fd43ccd25918c2846035 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Wed, 30 Aug 2023 19:48:39 +0530 Subject: [PATCH 6/7] dont write to hash if rewritten (#1227) --- .../blobbercore/filestore/storage.go | 22 +++++++++---------- .../handler/file_command_update.go | 12 +++++++--- .../handler/file_command_upload.go | 13 +++++++---- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 0a1781d13..2b686b7ab 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -88,17 +88,6 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i if err != nil { return nil, common.NewError("file_write_error", err.Error()) } - if !fileData.IsThumbnail { - _, err = f.Seek(fileData.UploadOffset, io.SeekStart) - if err != nil { - return nil, common.NewError("file_seek_error", err.Error()) - } - - _, err = io.CopyBuffer(fileData.Hasher, f, buf) - if err != nil { - return nil, common.NewError("file_read_error", err.Error()) - } - } finfo, err = f.Stat() if err != nil { @@ -109,6 +98,17 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i currentSize := finfo.Size() if currentSize > initialSize { // Is chunk new or rewritten + if !fileData.IsThumbnail { + _, err = f.Seek(fileData.UploadOffset, io.SeekStart) + if err != nil { + return nil, common.NewError("file_seek_error", err.Error()) + } + + _, err = io.CopyBuffer(fileData.Hasher, f, buf) + if err != nil { + return nil, common.NewError("file_read_error", err.Error()) + } + } fileRef.ChunkUploaded = true fs.updateAllocTempFileSize(allocID, currentSize-initialSize) } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index e62e2db4e..b5fd08714 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -87,14 +87,20 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, req *http.Requ result.Filename = cmd.fileChanger.Filename + isFinal := cmd.fileChanger.IsFinal + cmd.fileChanger.IsFinal = false + cmd.reloadChange(connectionObj) + if cmd.fileChanger.IsFinal { + return result, nil + } + cmd.fileChanger.IsFinal = isFinal + origfile, _, err := req.FormFile(UploadFile) if err != nil { return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) } defer origfile.Close() - cmd.reloadChange(connectionObj) - if cmd.fileChanger.Size == 0 { return result, common.NewError("invalid_parameters", "Invalid parameters. Size cannot be zero") } @@ -199,10 +205,10 @@ func (cmd *UpdateFileCommand) reloadChange(connectionObj *allocation.AllocationC } // reload uploaded size from db, it was chunk size from client - cmd.fileChanger.Size = dbFileChanger.Size cmd.fileChanger.ThumbnailFilename = dbFileChanger.ThumbnailFilename cmd.fileChanger.ThumbnailSize = dbFileChanger.ThumbnailSize cmd.fileChanger.ThumbnailHash = dbFileChanger.ThumbnailHash + cmd.fileChanger.IsFinal = dbFileChanger.IsFinal return } } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index d83e6b010..b319d53c1 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -103,15 +103,21 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, req *http.Requ return result, common.NewError("invalid_parameters", "Error Reading multi parts for file."+err.Error()) } defer origfile.Close() - + isFinal := cmd.fileChanger.IsFinal + cmd.fileChanger.IsFinal = false cmd.reloadChange(connectionObj) + if cmd.fileChanger.IsFinal { + result.Filename = cmd.fileChanger.Filename + return result, nil + } + cmd.fileChanger.IsFinal = isFinal var hasher *filestore.CommitHasher filePathHash := encryption.Hash(cmd.fileChanger.Path) if cmd.fileChanger.Size == 0 { return result, common.NewError("invalid_parameters", "Invalid parameters. Size cannot be zero") } - fmt.Println("cmd.fileChanger.Size", cmd.fileChanger.Size) + if cmd.fileChanger.UploadOffset == 0 { hasher = filestore.GetNewCommitHasher(cmd.fileChanger.Size) allocation.UpdateConnectionObjWithHasher(connectionObj.ID, filePathHash, hasher) @@ -207,11 +213,10 @@ func (cmd *UploadFileCommand) reloadChange(connectionObj *allocation.AllocationC logging.Logger.Error("reloadChange", zap.Error(err)) } - cmd.fileChanger.Size = dbChangeProcessor.Size cmd.fileChanger.ThumbnailFilename = dbChangeProcessor.ThumbnailFilename cmd.fileChanger.ThumbnailSize = dbChangeProcessor.ThumbnailSize cmd.fileChanger.ThumbnailHash = dbChangeProcessor.ThumbnailHash - + cmd.fileChanger.IsFinal = dbChangeProcessor.IsFinal return } } From 8004a14476718cfb6fdb9e6aa19c25d1a3c5993c Mon Sep 17 00:00:00 2001 From: Kishan Dhakan Date: Fri, 1 Sep 2023 22:41:08 +0100 Subject: [PATCH 7/7] update gosdk to staging --- go.mod | 7 +++++-- go.sum | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index df39f01df..faf9a96ea 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.8.17-0.20230620144453-ce90e5c28d5c + github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-openapi/runtime v0.26.0 @@ -42,7 +42,10 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc ) -require google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect +require ( + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect +) require github.com/pressly/goose/v3 v3.13.4 diff --git a/go.sum b/go.sum index 745928389..25c8fad34 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.8.17-0.20230620144453-ce90e5c28d5c h1:uFZKXY+1BXrOz4OojTM3I8/bwdKqRDD63C/iQCtLO9k= -github.com/0chain/gosdk v1.8.17-0.20230620144453-ce90e5c28d5c/go.mod h1:GgqNjoHBO0JN9TcpDAC28/VE2VsZi4MDr4qGdzfWV+I= +github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9 h1:GHTdYTmhNY9genBkNWLXdn3Z1yCtcbSNkcIFaKrqBRU= +github.com/0chain/gosdk v1.8.18-0.20230901213317-53d640a9b7f9/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= @@ -412,6 +412,8 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=