Merge branch 'sprint-1.10' into replace-with-multiops

.github/workflows/build-&-publish-docker-image.yml

@@ -21,6 +21,7 @@ env:
 
 jobs:
   blobber:
+    timeout-minutes: 25
     runs-on: [self-hosted, arc-runner]
     steps:
       - name: Set docker image tag
@@ -95,6 +96,7 @@ jobs:
           ./docker.local/bin/build.blobber.sh
 
   validator:
+    timeout-minutes: 20
     runs-on: [self-hosted, arc-runner]
     steps:
       - name: Set docker image tag 

code/go/0chain.net/blobbercore/allocation/allocationchange.go

@@ -217,7 +217,6 @@ func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, allocatio
 			return err
 		}
 	}
-	logging.Logger.Info("ApplyChanges", zap.Any("rootRef", rootRef))
 	_, err = rootRef.CalculateHash(ctx, true)
 	return err
 }

code/go/0chain.net/blobbercore/allocation/entity.go

@@ -48,13 +48,14 @@ type Allocation struct {
 	RepairerID     string           `gorm:"column:repairer_id;size:64;not null"`
 	Expiration     common.Timestamp `gorm:"column:expiration_date;not null"`
 	// AllocationRoot allcation_root of last write_marker
-	AllocationRoot   string        `gorm:"column:allocation_root;size:64;not null;default:''"`
-	FileMetaRoot     string        `gorm:"column:file_meta_root;size:64;not null;default:''"`
-	BlobberSize      int64         `gorm:"column:blobber_size;not null;default:0"`
-	BlobberSizeUsed  int64         `gorm:"column:blobber_size_used;not null;default:0"`
-	LatestRedeemedWM string        `gorm:"column:latest_redeemed_write_marker;size:64"`
-	IsRedeemRequired bool          `gorm:"column:is_redeem_required"`
-	TimeUnit         time.Duration `gorm:"column:time_unit;not null;default:172800000000000"`
+	AllocationRoot   string           `gorm:"column:allocation_root;size:64;not null;default:''"`
+	FileMetaRoot     string           `gorm:"column:file_meta_root;size:64;not null;default:''"`
+	BlobberSize      int64            `gorm:"column:blobber_size;not null;default:0"`
+	BlobberSizeUsed  int64            `gorm:"column:blobber_size_used;not null;default:0"`
+	LatestRedeemedWM string           `gorm:"column:latest_redeemed_write_marker;size:64"`
+	IsRedeemRequired bool             `gorm:"column:is_redeem_required"`
+	TimeUnit         time.Duration    `gorm:"column:time_unit;not null;default:172800000000000"`
+	StartTime        common.Timestamp `gorm:"column:start_time;not null"`
 	// Ending and cleaning
 	CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"`
 	Finalized bool `gorm:"column:finalized;not null;default:false"`

code/go/0chain.net/blobbercore/allocation/protocol.go

@@ -108,6 +108,7 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc
 	a.Finalized = sa.Finalized
 	a.TimeUnit = sa.TimeUnit
 	a.FileOptions = sa.FileOptions
+	a.StartTime = sa.StartTime
 
 	m := map[string]interface{}{
 		"allocation_id":  a.ID,

code/go/0chain.net/blobbercore/filestore/storage.go

@@ -83,8 +83,8 @@ func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, i
 	if err != nil {
 		return nil, common.NewError("file_seek_error", err.Error())
 	}
-
-	writtenSize, err := io.Copy(f, infile)
+	buf := make([]byte, BufferSize)
+	writtenSize, err := io.CopyBuffer(f, infile, buf)
 	if err != nil {
 		return nil, common.NewError("file_write_error", err.Error())
 	}

code/go/0chain.net/blobbercore/handler/file_command_upload.go

@@ -50,7 +50,6 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request
 	}
 
 	fileChanger := &allocation.UploadFileChanger{}
-
 	uploadMetaString := req.FormValue(UploadMeta)
 	err := json.Unmarshal([]byte(uploadMetaString), fileChanger)
 	if err != nil {

code/go/0chain.net/blobbercore/handler/object_operation_handler.go

@@ -490,19 +490,19 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b
 		return nil, common.NewError("invalid_parameters", "Invalid connection id passed")
 	}
 
-	err = checkPendingMarkers(ctx, allocationObj.ID)
-	if err != nil {
-		Logger.Error("Error checking pending markers", zap.Error(err))
-		return nil, common.NewError("pending_markers", "previous marker is still pending to be redeemed")
-	}
-
 	// Lock will compete with other CommitWrites and Challenge validation
 	mutex := lock.GetMutex(allocationObj.TableName(), allocationID)
 	mutex.Lock()
 	defer mutex.Unlock()
 
 	elapsedGetLock := time.Since(startTime) - elapsedAllocation
 
+	err = checkPendingMarkers(ctx, allocationObj.ID)
+	if err != nil {
+		Logger.Error("Error checking pending markers", zap.Error(err))
+		return nil, common.NewError("pending_markers", "previous marker is still pending to be redeemed")
+	}
+
 	connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID)
 
 	if err != nil {

code/go/0chain.net/blobbercore/readmarker/readmarker.go

@@ -103,8 +103,11 @@ func (rm *ReadMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.All
 		return common.NewError("read_marker_validation_failed", "Read Marker clientID does not match request clientID")
 	}
 
+	if rm.LatestRM.Timestamp < sa.StartTime {
+		return common.NewError("read_marker_validation_failed", "Readmarker timestamp is before the allocation start time")
+	}
+
 	if rm.LatestRM.Timestamp > sa.Expiration {
-		zLogger.Logger.Error("Readmarker is for an expired allocation", zap.Any("rm", rm))
 		return common.NewError("read_marker_validation_failed", "Readmarker is for an expired allocation")
 	}
 

code/go/0chain.net/blobbercore/writemarker/protocol.go

@@ -81,10 +81,13 @@ func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *al
 	if clientID == "" || clientID != wme.WM.ClientID || clientID != co.ClientID || co.ClientID != wme.WM.ClientID {
 		return common.NewError("write_marker_validation_failed", "Write Marker is not by the same client who uploaded")
 	}
+	if wme.WM.Timestamp < dbAllocation.StartTime {
+		return common.NewError("write_marker_validation_failed", "Write Marker timestamp is before the allocation start time")
+	}
 
 	currTime := common.Now()
 	// blobber clock is allowed to be 10 seconds behind the current time
-	if wme.WM.Timestamp > currTime+10 {
+	if wme.WM.Timestamp > currTime+60 {
 		return common.NewError("write_marker_validation_failed", "Write Marker timestamp is in the future")
 	}
 

code/go/0chain.net/core/transaction/entity.go

@@ -90,6 +90,7 @@ type StorageAllocation struct {
 	TimeUnit       time.Duration        `json:"time_unit"`
 	WritePool      uint64               `json:"write_pool"`
 	FileOptions    uint16               `json:"file_options"`
+	StartTime      common.Timestamp     `json:"start_time"`
 
 	DataShards   int64 `json:"data_shards"`
 	ParityShards int64 `json:"parity_shards"`

goose/migrations/001_blobber_meta.sql

@@ -89,7 +89,8 @@ CREATE TABLE public.allocations (
     time_unit bigint DEFAULT '172800000000000'::bigint NOT NULL,
     cleaned_up boolean DEFAULT false NOT NULL,
     finalized boolean DEFAULT false NOT NULL,
-    file_options integer DEFAULT 63 NOT NULL
+    file_options integer DEFAULT 63 NOT NULL,
+    start_time bigint NOT NULL
 );