diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index e6322feec..a91093587 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -21,6 +21,7 @@ env: jobs: blobber: + timeout-minutes: 25 runs-on: [self-hosted, arc-runner] steps: - name: Set docker image tag @@ -95,6 +96,7 @@ jobs: ./docker.local/bin/build.blobber.sh validator: + timeout-minutes: 20 runs-on: [self-hosted, arc-runner] steps: - name: Set docker image tag diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index ba39c8a8c..4226c41f6 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -217,7 +217,6 @@ func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, allocatio return err } } - logging.Logger.Info("ApplyChanges", zap.Any("rootRef", rootRef)) _, err = rootRef.CalculateHash(ctx, true) return err } diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go index 4530a8322..4022e813f 100644 --- a/code/go/0chain.net/blobbercore/allocation/entity.go +++ b/code/go/0chain.net/blobbercore/allocation/entity.go @@ -48,13 +48,14 @@ type Allocation struct { RepairerID string `gorm:"column:repairer_id;size:64;not null"` Expiration common.Timestamp `gorm:"column:expiration_date;not null"` // AllocationRoot allcation_root of last write_marker - AllocationRoot string `gorm:"column:allocation_root;size:64;not null;default:''"` - FileMetaRoot string `gorm:"column:file_meta_root;size:64;not null;default:''"` - BlobberSize int64 `gorm:"column:blobber_size;not null;default:0"` - BlobberSizeUsed int64 `gorm:"column:blobber_size_used;not null;default:0"` - LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker;size:64"` - IsRedeemRequired bool `gorm:"column:is_redeem_required"` - TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"` + AllocationRoot string `gorm:"column:allocation_root;size:64;not null;default:''"` + FileMetaRoot string `gorm:"column:file_meta_root;size:64;not null;default:''"` + BlobberSize int64 `gorm:"column:blobber_size;not null;default:0"` + BlobberSizeUsed int64 `gorm:"column:blobber_size_used;not null;default:0"` + LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker;size:64"` + IsRedeemRequired bool `gorm:"column:is_redeem_required"` + TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"` + StartTime common.Timestamp `gorm:"column:start_time;not null"` // Ending and cleaning CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"` Finalized bool `gorm:"column:finalized;not null;default:false"` diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index 7edd76cfa..b38d0626b 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -108,6 +108,7 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc a.Finalized = sa.Finalized a.TimeUnit = sa.TimeUnit a.FileOptions = sa.FileOptions + a.StartTime = sa.StartTime m := map[string]interface{}{ "allocation_id": a.ID, diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 3d3864a84..56cc92979 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -83,8 +83,8 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i if err != nil { return nil, common.NewError("file_seek_error", err.Error()) } - - writtenSize, err := io.Copy(f, infile) + buf := make([]byte, BufferSize) + writtenSize, err := io.CopyBuffer(f, infile, buf) if err != nil { return nil, common.NewError("file_write_error", err.Error()) } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index f22c85948..101d5ef79 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -50,7 +50,6 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request } fileChanger := &allocation.UploadFileChanger{} - uploadMetaString := req.FormValue(UploadMeta) err := json.Unmarshal([]byte(uploadMetaString), fileChanger) if err != nil { diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index c82c52a5b..59d6a1bd1 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -490,12 +490,6 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b return nil, common.NewError("invalid_parameters", "Invalid connection id passed") } - err = checkPendingMarkers(ctx, allocationObj.ID) - if err != nil { - Logger.Error("Error checking pending markers", zap.Error(err)) - return nil, common.NewError("pending_markers", "previous marker is still pending to be redeemed") - } - // Lock will compete with other CommitWrites and Challenge validation mutex := lock.GetMutex(allocationObj.TableName(), allocationID) mutex.Lock() @@ -503,6 +497,12 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b elapsedGetLock := time.Since(startTime) - elapsedAllocation + err = checkPendingMarkers(ctx, allocationObj.ID) + if err != nil { + Logger.Error("Error checking pending markers", zap.Error(err)) + return nil, common.NewError("pending_markers", "previous marker is still pending to be redeemed") + } + connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) if err != nil { diff --git a/code/go/0chain.net/blobbercore/readmarker/readmarker.go b/code/go/0chain.net/blobbercore/readmarker/readmarker.go index 7d90819ef..4522f7129 100644 --- a/code/go/0chain.net/blobbercore/readmarker/readmarker.go +++ b/code/go/0chain.net/blobbercore/readmarker/readmarker.go @@ -103,8 +103,11 @@ func (rm *ReadMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.All return common.NewError("read_marker_validation_failed", "Read Marker clientID does not match request clientID") } + if rm.LatestRM.Timestamp < sa.StartTime { + return common.NewError("read_marker_validation_failed", "Readmarker timestamp is before the allocation start time") + } + if rm.LatestRM.Timestamp > sa.Expiration { - zLogger.Logger.Error("Readmarker is for an expired allocation", zap.Any("rm", rm)) return common.NewError("read_marker_validation_failed", "Readmarker is for an expired allocation") } diff --git a/code/go/0chain.net/blobbercore/writemarker/protocol.go b/code/go/0chain.net/blobbercore/writemarker/protocol.go index bca1b19e2..cfe950b40 100644 --- a/code/go/0chain.net/blobbercore/writemarker/protocol.go +++ b/code/go/0chain.net/blobbercore/writemarker/protocol.go @@ -81,10 +81,13 @@ func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *al if clientID == "" || clientID != wme.WM.ClientID || clientID != co.ClientID || co.ClientID != wme.WM.ClientID { return common.NewError("write_marker_validation_failed", "Write Marker is not by the same client who uploaded") } + if wme.WM.Timestamp < dbAllocation.StartTime { + return common.NewError("write_marker_validation_failed", "Write Marker timestamp is before the allocation start time") + } currTime := common.Now() // blobber clock is allowed to be 10 seconds behind the current time - if wme.WM.Timestamp > currTime+10 { + if wme.WM.Timestamp > currTime+60 { return common.NewError("write_marker_validation_failed", "Write Marker timestamp is in the future") } diff --git a/code/go/0chain.net/core/transaction/entity.go b/code/go/0chain.net/core/transaction/entity.go index 43fc030f5..3d096d5a0 100644 --- a/code/go/0chain.net/core/transaction/entity.go +++ b/code/go/0chain.net/core/transaction/entity.go @@ -90,6 +90,7 @@ type StorageAllocation struct { TimeUnit time.Duration `json:"time_unit"` WritePool uint64 `json:"write_pool"` FileOptions uint16 `json:"file_options"` + StartTime common.Timestamp `json:"start_time"` DataShards int64 `json:"data_shards"` ParityShards int64 `json:"parity_shards"` diff --git a/goose/migrations/001_blobber_meta.sql b/goose/migrations/001_blobber_meta.sql index e4d2158ba..86b3883d0 100644 --- a/goose/migrations/001_blobber_meta.sql +++ b/goose/migrations/001_blobber_meta.sql @@ -89,7 +89,8 @@ CREATE TABLE public.allocations ( time_unit bigint DEFAULT '172800000000000'::bigint NOT NULL, cleaned_up boolean DEFAULT false NOT NULL, finalized boolean DEFAULT false NOT NULL, - file_options integer DEFAULT 63 NOT NULL + file_options integer DEFAULT 63 NOT NULL, + start_time bigint NOT NULL );