diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index 7c9d6628c..ab85ede34 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -206,8 +206,8 @@ func CreateChunkedUpload( } su.loadProgress() - - su.fileHasher = CreateHasher(getShardSize(su.fileMeta.ActualSize, su.allocationObj.DataShards, su.encryptOnUpload)) + su.shardSize = getShardSize(su.fileMeta.ActualSize, su.allocationObj.DataShards, su.encryptOnUpload) + su.fileHasher = CreateHasher(su.shardSize) // encrypt option has been changed. upload it from scratch // chunkSize has been changed. upload it from scratch @@ -324,6 +324,8 @@ type ChunkedUpload struct { shardUploadedSize int64 // shardUploadedThumbnailSize how much thumbnail bytes a shard has. it is original size shardUploadedThumbnailSize int64 + // size of shard + shardSize int64 // statusCallback trigger progress on StatusCallback statusCallback StatusCallback @@ -395,7 +397,7 @@ func (su *ChunkedUpload) createUploadProgress(connectionId string) { for i := 0; i < len(su.progress.Blobbers); i++ { su.progress.Blobbers[i] = &UploadBlobberStatus{ - Hasher: CreateHasher(getShardSize(su.fileMeta.ActualSize, su.allocationObj.DataShards, su.encryptOnUpload)), + Hasher: CreateHasher(su.shardSize), } } @@ -626,7 +628,6 @@ func (su *ChunkedUpload) readChunks(num int) (*batchChunksData, error) { func (su *ChunkedUpload) processUpload(chunkStartIndex, chunkEndIndex int, fileShards []blobberShards, thumbnailShards blobberShards, isFinal bool, uploadLength int64) error { - su.consensus.Reset() ctx, cancel := context.WithCancel(context.TODO()) @@ -663,7 +664,7 @@ func (su *ChunkedUpload) processUpload(chunkStartIndex, chunkEndIndex int, body, formData, err := su.formBuilder.Build( &su.fileMeta, blobber.progress.Hasher, su.progress.ConnectionID, su.chunkSize, chunkStartIndex, chunkEndIndex, isFinal, encryptedKey, - fileShards[pos], thumbnailChunkData) + fileShards[pos], thumbnailChunkData, su.shardSize) if err != nil { errC := atomic.AddInt32(&errCount, 1) if errC > int32(su.allocationObj.ParityShards-1) { // If atleast data shards + 1 number of blobbers can process the upload, it can be repaired later diff --git a/zboxcore/sdk/chunked_upload_form_builder.go b/zboxcore/sdk/chunked_upload_form_builder.go index c8fa20fdf..55515b8f8 100644 --- a/zboxcore/sdk/chunked_upload_form_builder.go +++ b/zboxcore/sdk/chunked_upload_form_builder.go @@ -22,7 +22,7 @@ type ChunkedUploadFormBuilder interface { fileMeta *FileMeta, hasher Hasher, connectionID string, chunkSize int64, chunkStartIndex, chunkEndIndex int, isFinal bool, encryptedKey string, fileChunksData [][]byte, - thumbnailChunkData []byte, + thumbnailChunkData []byte, shardSize int64, ) (*bytes.Buffer, ChunkedUploadFormMetadata, error) } @@ -48,7 +48,7 @@ func (b *chunkedUploadFormBuilder) Build( fileMeta *FileMeta, hasher Hasher, connectionID string, chunkSize int64, chunkStartIndex, chunkEndIndex int, isFinal bool, encryptedKey string, fileChunksData [][]byte, - thumbnailChunkData []byte, + thumbnailChunkData []byte, shardSize int64, ) (*bytes.Buffer, ChunkedUploadFormMetadata, error) { metadata := ChunkedUploadFormMetadata{ @@ -78,6 +78,7 @@ func (b *chunkedUploadFormBuilder) Build( ChunkStartIndex: chunkStartIndex, ChunkEndIndex: chunkEndIndex, UploadOffset: chunkSize * int64(chunkStartIndex), + Size: shardSize, } formWriter := multipart.NewWriter(body) diff --git a/zboxcore/sdk/chunked_upload_form_builder_bench_test.go b/zboxcore/sdk/chunked_upload_form_builder_bench_test.go index d4fbcf1b9..25cea9953 100644 --- a/zboxcore/sdk/chunked_upload_form_builder_bench_test.go +++ b/zboxcore/sdk/chunked_upload_form_builder_bench_test.go @@ -68,7 +68,7 @@ func BenchmarkChunkedUploadFormBuilder(b *testing.B) { fileBytes := buf[begin:end] - _, _, err := builder.Build(fileMeta, hasher, "connectionID", int64(bm.ChunkSize), chunkIndex, chunkIndex, isFinal, "", [][]byte{fileBytes}, nil) + _, _, err := builder.Build(fileMeta, hasher, "connectionID", int64(bm.ChunkSize), chunkIndex, chunkIndex, isFinal, "", [][]byte{fileBytes}, nil, getShardSize(fileMeta.ActualSize, 1, false)) if err != nil { b.Fatal(err) return diff --git a/zboxcore/sdk/chunked_upload_model.go b/zboxcore/sdk/chunked_upload_model.go index 1a7f96396..33bfd9d52 100644 --- a/zboxcore/sdk/chunked_upload_model.go +++ b/zboxcore/sdk/chunked_upload_model.go @@ -78,6 +78,7 @@ type UploadFormData struct { ChunkEndIndex int `json:"chunk_end_index,omitempty"` // end index of chunks. all chunks MUST be uploaded one by one because of streaming merkle hash ChunkSize int64 `json:"chunk_size,omitempty"` // the size of a chunk. 64*1024 is default UploadOffset int64 `json:"upload_offset,omitempty"` // It is next position that new incoming chunk should be append to + Size int64 `json:"size"` // total size of shard }