From f8d40d291844d486aded020796f24267a137b5e1 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:43:36 +0530 Subject: [PATCH 01/13] Revert "Revert "Session key"" --- .../blobbercore/allocation/entity.go | 5 ++- .../allocation/file_changer_base.go | 5 ++- .../allocation/file_changer_update.go | 2 + .../allocation/file_changer_upload.go | 2 + .../blobbercore/allocation/protocol.go | 2 +- .../blobbercore/allocation/workers.go | 3 ++ .../0chain.net/blobbercore/allocation/zcn.go | 2 + .../blobbercore/challenge/protocol.go | 6 ++- .../handler/file_command_update.go | 10 ++++- .../handler/file_command_upload.go | 10 ++++- .../0chain.net/blobbercore/reference/ref.go | 4 ++ code/go/0chain.net/core/encryption/keys.go | 36 +++++++++++++++-- code/go/0chain.net/core/transaction/entity.go | 29 +++++++------- .../storage/challenge_handler.go | 40 ++++++++++++------- .../validatorcore/storage/models.go | 33 +++++++++------ .../validatorcore/storage/models_test.go | 2 + goose/migrations/1730098482_session_key.sql | 7 ++++ 17 files changed, 144 insertions(+), 54 deletions(-) create mode 100644 goose/migrations/1730098482_session_key.sql diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go index 74a223be5..30fece0e5 100644 --- a/code/go/0chain.net/blobbercore/allocation/entity.go +++ b/code/go/0chain.net/blobbercore/allocation/entity.go @@ -62,8 +62,9 @@ type Allocation struct { TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"` StartTime common.Timestamp `gorm:"column:start_time;not null"` // Ending and cleaning - CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"` - Finalized bool `gorm:"column:finalized;not null;default:false"` + CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"` + Finalized bool `gorm:"column:finalized;not null;default:false"` + OwnerSigningPublicKey string `gorm:"column:owner_signing_public_key;size:512;not null" json:"owner_signing_public_key"` // FileOptions to define file restrictions on an allocation for third-parties // default 00000000 for all crud operations suggesting only owner has the below listed abilities. diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index 1d86e30d1..20345b565 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -52,8 +52,9 @@ type BaseFileChanger struct { EncryptedKeyPoint string `json:"encrypted_key_point,omitempty"` CustomMeta string `json:"custom_meta,omitempty"` - ChunkSize int64 `json:"chunk_size,omitempty"` // the size of achunk. 64*1024 is default - IsFinal bool `json:"is_final,omitempty"` // current chunk is last or not + ChunkSize int64 `json:"chunk_size,omitempty"` // the size of achunk. 64*1024 is default + IsFinal bool `json:"is_final,omitempty"` // current chunk is last or not + SignatureVersion int `json:"signature_version,omitempty"` ChunkStartIndex int `json:"chunk_start_index,omitempty"` // start index of chunks. ChunkEndIndex int `json:"chunk_end_index,omitempty"` // end index of chunks. all chunks MUST be uploaded one by one because of CompactMerkleTree diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index 590d25d60..a1b918f00 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -106,6 +106,7 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference fileRef.ChunkSize = nf.ChunkSize fileRef.IsPrecommit = true fileRef.FilestoreVersion = filestore.VERSION + fileRef.SignatureVersion = nf.SignatureVersion return rootRef, nil } @@ -173,6 +174,7 @@ func (nf *UpdateFileChanger) ApplyChangeV2(ctx context.Context, allocationRoot, PathLevel: len(strings.Split(strings.TrimRight(nf.Path, "/"), "/")), NumBlocks: int64(math.Ceil(float64(nf.Size*1.0) / float64(nf.ChunkSize))), NumUpdates: refResult.NumUpdates + 1, + SignatureVersion: nf.SignatureVersion, } nf.storageVersion = 1 newFile.FileMetaHash = encryption.Hash(newFile.GetFileMetaHashDataV2()) diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 59936dd10..4acb713c7 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -130,6 +130,7 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference HashToBeComputed: true, IsPrecommit: true, FilestoreVersion: filestore.VERSION, + SignatureVersion: nf.SignatureVersion, } fileID, ok := fileIDMeta[newFile.Path] @@ -197,6 +198,7 @@ func (nf *UploadFileChanger) ApplyChangeV2(ctx context.Context, allocationRoot, PathLevel: len(strings.Split(strings.TrimRight(nf.Path, "/"), "/")), NumBlocks: int64(math.Ceil(float64(nf.Size*1.0) / float64(nf.ChunkSize))), NumUpdates: 1, + SignatureVersion: nf.SignatureVersion, } nf.storageVersion = 1 newFile.FileMetaHash = encryption.Hash(newFile.GetFileMetaHashDataV2()) diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index d7c53fd0e..386a05485 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -110,8 +110,8 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc a.TimeUnit = sa.TimeUnit a.FileOptions = sa.FileOptions a.StartTime = sa.StartTime - // Only for testing purpose a.StorageVersion = uint8(sa.StorageVersion) + a.OwnerSigningPublicKey = sa.OwnerSigningPublicKey m := map[string]interface{}{ "allocation_id": a.ID, diff --git a/code/go/0chain.net/blobbercore/allocation/workers.go b/code/go/0chain.net/blobbercore/allocation/workers.go index cf70769db..3eb535869 100644 --- a/code/go/0chain.net/blobbercore/allocation/workers.go +++ b/code/go/0chain.net/blobbercore/allocation/workers.go @@ -246,6 +246,7 @@ func updateAllocationInDB(ctx context.Context, a *Allocation, sa *transaction.St a.Tx = sa.Tx a.OwnerID = sa.OwnerID a.OwnerPublicKey = sa.OwnerPublicKey + a.OwnerSigningPublicKey = sa.OwnerSigningPublicKey // // update fields a.Expiration = sa.Expiration @@ -263,6 +264,7 @@ func updateAllocationInDB(ctx context.Context, a *Allocation, sa *transaction.St updateMap["finalized"] = a.Finalized updateMap["file_options"] = a.FileOptions updateMap["blobber_size"] = a.BlobberSize + updateMap["owner_signing_public_key"] = a.OwnerSigningPublicKey updateOption := func(alloc *Allocation) { alloc.Tx = a.Tx @@ -273,6 +275,7 @@ func updateAllocationInDB(ctx context.Context, a *Allocation, sa *transaction.St alloc.Finalized = a.Finalized alloc.FileOptions = a.FileOptions alloc.BlobberSize = a.BlobberSize + alloc.OwnerSigningPublicKey = a.OwnerSigningPublicKey } // update terms diff --git a/code/go/0chain.net/blobbercore/allocation/zcn.go b/code/go/0chain.net/blobbercore/allocation/zcn.go index c5d419ecf..f2f7cb1d0 100644 --- a/code/go/0chain.net/blobbercore/allocation/zcn.go +++ b/code/go/0chain.net/blobbercore/allocation/zcn.go @@ -52,6 +52,8 @@ func SyncAllocation(allocationId string) (*Allocation, error) { alloc.Finalized = sa.Finalized alloc.TimeUnit = sa.TimeUnit alloc.FileOptions = sa.FileOptions + alloc.StorageVersion = uint8(sa.StorageVersion) + alloc.OwnerSigningPublicKey = sa.OwnerSigningPublicKey // related terms terms := make([]*Terms, 0, len(sa.BlobberDetails)) diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index 71e7a2adf..53540e827 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -5,13 +5,14 @@ import ( "encoding/hex" "encoding/json" "errors" - "github.com/0chain/blobber/code/go/0chain.net/core/transaction" - coreTxn "github.com/0chain/gosdk/core/transaction" "math/rand" "strings" "sync" "time" + "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + coreTxn "github.com/0chain/gosdk/core/transaction" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" @@ -366,6 +367,7 @@ func (cr *ChallengeEntity) getPostDataV2(ctx context.Context, allocationObj *all FixedMerkleRoot: ref.FixedMerkleRoot, Size: ref.Size, FileMetaHash: ref.FileMetaHash, + SignatureVersion: ref.SignatureVersion, } postData["meta"] = metaRef } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index 9d8e8d5b2..7b35854d1 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -155,7 +155,15 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj hashData := fmt.Sprintf("%s:%s:%s:%s", cmd.fileChanger.ActualHash, cmd.fileChanger.ValidationRoot, cmd.fileChanger.FixedMerkleRoot, node.Self.ID) hash = encryption.Hash(hashData) } - verify, err := encryption.Verify(allocationObj.OwnerPublicKey, cmd.fileChanger.ValidationRootSignature, hash) + var ( + err error + verify bool + ) + if cmd.fileChanger.SignatureVersion == reference.SignatureV2 { + verify, err = encryption.VerifyEd25519(allocationObj.OwnerSigningPublicKey, cmd.fileChanger.ValidationRootSignature, hash) + } else { + verify, err = encryption.Verify(allocationObj.OwnerPublicKey, cmd.fileChanger.ValidationRootSignature, hash) + } if err != nil || !verify { logging.Logger.Error("UpdateFileCommand.VerifySignature", zap.Error(err)) return result, common.NewError("update_error", "Failed to verify validation root signature. ") diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index 3c62526c9..279e67701 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -167,7 +167,15 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj hashData := fmt.Sprintf("%s:%s:%s:%s", cmd.fileChanger.ActualHash, cmd.fileChanger.ValidationRoot, cmd.fileChanger.FixedMerkleRoot, node.Self.ID) hash = encryption.Hash(hashData) } - verify, err := encryption.Verify(allocationObj.OwnerPublicKey, cmd.fileChanger.ValidationRootSignature, hash) + var ( + err error + verify bool + ) + if cmd.fileChanger.SignatureVersion == reference.SignatureV2 { + verify, err = encryption.VerifyEd25519(allocationObj.OwnerSigningPublicKey, cmd.fileChanger.ValidationRootSignature, hash) + } else { + verify, err = encryption.Verify(allocationObj.OwnerPublicKey, cmd.fileChanger.ValidationRootSignature, hash) + } if err != nil || !verify { logging.Logger.Error("UploadFileCommand.VerifySignature", zap.Error(err)) return result, common.NewError("upload_error", "Failed to verify validation root signature. ") diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 13aa4c47d..8c3cc0709 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -28,6 +28,7 @@ const ( DIR_LIST_TAG = "dirlist" FILE_LIST_TAG = "filelist" + SignatureV2 = 1 ) var ( @@ -91,6 +92,7 @@ type Ref struct { NumUpdates int64 `gorm:"column:num_of_updates" json:"num_of_updates"` NumBlockDownloads int64 `gorm:"column:num_of_block_downloads" json:"num_of_block_downloads"` FilestoreVersion int `gorm:"column:filestore_version" json:"-"` + SignatureVersion int `gorm:"column:signature_version" json:"signature_version" filelist:"signature_version"` IsEmpty bool `gorm:"-" dirlist:"is_empty"` HashToBeComputed bool `gorm:"-"` prevID int64 `gorm:"-"` @@ -148,6 +150,7 @@ type PaginatedRef struct { //Gorm smart select fields. EncryptedKey string `gorm:"column:encrypted_key" json:"encrypted_key,omitempty"` EncryptedKeyPoint string `gorm:"column:encrypted_key_point" json:"encrypted_key_point,omitempty"` FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"` + SignatureVersion int `gorm:"column:signature_version" json:"signature_version,omitempty" filelist:"signature_version"` CreatedAt common.Timestamp `gorm:"column:created_at" json:"created_at,omitempty"` UpdatedAt common.Timestamp `gorm:"column:updated_at" json:"updated_at,omitempty"` @@ -166,6 +169,7 @@ type RefMeta struct { FixedMerkleRoot string `json:"fixed_merkle_root"` Size int64 `json:"size"` FileMetaHash string `json:"file_meta_hash"` + SignatureVersion int `json:"signature_version"` } // GetReferenceLookup hash(allocationID + ":" + path) diff --git a/code/go/0chain.net/core/encryption/keys.go b/code/go/0chain.net/core/encryption/keys.go index bb652f1bf..e07ea647d 100644 --- a/code/go/0chain.net/core/encryption/keys.go +++ b/code/go/0chain.net/core/encryption/keys.go @@ -2,6 +2,8 @@ package encryption import ( "bufio" + "crypto/ed25519" + "encoding/hex" "io" "strings" @@ -13,8 +15,10 @@ import ( "github.com/herumi/bls-go-binary/bls" ) -/*ReadKeys - reads a publicKey and a privateKey from a Reader. -They are assumed to be in two separate lines one followed by the other*/ +/* +ReadKeys - reads a publicKey and a privateKey from a Reader. +They are assumed to be in two separate lines one followed by the other +*/ func ReadKeys(reader io.Reader) (publicKey, privateKey, publicIp, port string) { scanner := bufio.NewScanner(reader) scanner.Scan() @@ -44,14 +48,38 @@ func Verify(publicKey, signature, hash string) (bool, error) { return false, common.NewError("invalid_signature_scheme", "Invalid signature scheme. Please check configuration") } +// VerifyEd25519 - verify the signature using the public key +func VerifyEd25519(publicKey, signature, hash string) (bool, error) { + if len(publicKey) == 0 { + return false, common.NewError("invalid_public_key", "Invalid public key") + } + if len(signature) == 0 { + return false, common.NewError("invalid_signature", "Invalid signature") + } + sig, err := hex.DecodeString(signature) + if err != nil { + return false, err + } + pub, err := hex.DecodeString(publicKey) + if err != nil { + return false, err + } + msg, err := hex.DecodeString(hash) + if err != nil { + return false, err + } + return ed25519.Verify(pub, msg, sig), nil +} + // If input is normal herumi/bls public key, it returns it immmediately. -// So this is completely backward compatible with herumi/bls. +// +// So this is completely backward compatible with herumi/bls. +// // If input is MIRACL public key, convert it to herumi/bls public key. // // This is an example of the raw public key we expect from MIRACL var miraclExamplePK = `0418a02c6bd223ae0dfda1d2f9a3c81726ab436ce5e9d17c531ff0a385a13a0b491bdfed3a85690775ee35c61678957aaba7b1a1899438829f1dc94248d87ed36817f6dfafec19bfa87bf791a4d694f43fec227ae6f5a867490e30328cac05eaff039ac7dfc3364e851ebd2631ea6f1685609fc66d50223cc696cb59ff2fee47ac` -// // This is an example of the same MIRACL public key serialized with ToString(). // pk ([1bdfed3a85690775ee35c61678957aaba7b1a1899438829f1dc94248d87ed368,18a02c6bd223ae0dfda1d2f9a3c81726ab436ce5e9d17c531ff0a385a13a0b49],[039ac7dfc3364e851ebd2631ea6f1685609fc66d50223cc696cb59ff2fee47ac,17f6dfafec19bfa87bf791a4d694f43fec227ae6f5a867490e30328cac05eaff]) func MiraclToHerumiPK(pk string) string { diff --git a/code/go/0chain.net/core/transaction/entity.go b/code/go/0chain.net/core/transaction/entity.go index a204d3f70..0bc02005b 100644 --- a/code/go/0chain.net/core/transaction/entity.go +++ b/code/go/0chain.net/core/transaction/entity.go @@ -53,20 +53,21 @@ type BlobberAllocation struct { } type StorageAllocation struct { - ID string `json:"id"` - Tx string `json:"tx"` - OwnerPublicKey string `json:"owner_public_key"` - OwnerID string `json:"owner_id"` - Size int64 `json:"size"` - UsedSize int64 `json:"used_size"` - Expiration common.Timestamp `json:"expiration_date"` - BlobberDetails []*BlobberAllocation `json:"blobber_details"` - Finalized bool `json:"finalized"` - TimeUnit time.Duration `json:"time_unit"` - WritePool uint64 `json:"write_pool"` - FileOptions uint16 `json:"file_options"` - StartTime common.Timestamp `json:"start_time"` - StorageVersion int `json:"storage_version"` + ID string `json:"id"` + Tx string `json:"tx"` + OwnerPublicKey string `json:"owner_public_key"` + OwnerID string `json:"owner_id"` + Size int64 `json:"size"` + UsedSize int64 `json:"used_size"` + Expiration common.Timestamp `json:"expiration_date"` + BlobberDetails []*BlobberAllocation `json:"blobber_details"` + Finalized bool `json:"finalized"` + TimeUnit time.Duration `json:"time_unit"` + WritePool uint64 `json:"write_pool"` + FileOptions uint16 `json:"file_options"` + StartTime common.Timestamp `json:"start_time"` + StorageVersion int `json:"storage_version"` + OwnerSigningPublicKey string `json:"owner_signing_public_key"` DataShards int64 `json:"data_shards"` ParityShards int64 `json:"parity_shards"` diff --git a/code/go/0chain.net/validatorcore/storage/challenge_handler.go b/code/go/0chain.net/validatorcore/storage/challenge_handler.go index d4c8b4bf9..448db4e3d 100644 --- a/code/go/0chain.net/validatorcore/storage/challenge_handler.go +++ b/code/go/0chain.net/validatorcore/storage/challenge_handler.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "encoding/json" - "github.com/0chain/gosdk/zboxcore/sdk" "io" "net/http" "strings" @@ -14,6 +13,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" + "github.com/0chain/blobber/code/go/0chain.net/core/transaction" "go.uber.org/zap" "golang.org/x/crypto/sha3" @@ -42,14 +42,12 @@ func challengeHandler(ctx context.Context, r *http.Request) (interface{}, error) time.Sleep(1 * time.Second) - sdkAlloc, err := sdk.GetAllocation(challengeObj.AllocationID) + allocationObj, err := requestAllocation(challengeObj.AllocationID) if err != nil { logging.Logger.Error("Error getting allocation from chain", zap.String("allocation_id", challengeObj.AllocationID), zap.Error(err)) return nil, common.NewError("invalid_parameters", "Allocation could not be verified. "+err.Error()) } - allocationObj := sdkAllocToBlobberAlloc(sdkAlloc) - err = challengeRequest.VerifyChallenge(challengeObj, allocationObj) if err != nil { updateStats(false) @@ -61,17 +59,31 @@ func challengeHandler(ctx context.Context, r *http.Request) (interface{}, error) return ValidValidationTicket(challengeObj, challengeRequest.ChallengeID, challengeHash) } -func sdkAllocToBlobberAlloc(sdkAlloc *sdk.Allocation) *Allocation { - return &Allocation{ - ID: sdkAlloc.ID, - DataShards: sdkAlloc.DataShards, - ParityShards: sdkAlloc.ParityShards, - Size: sdkAlloc.Size, - Owner: sdkAlloc.Owner, - OwnerPublicKey: sdkAlloc.OwnerPublicKey, - UsedSize: sdkAlloc.Stats.UsedSize, - Expiration: common.Timestamp(sdkAlloc.Expiration), +func requestAllocation(allocID string) (allocation *Allocation, err error) { + var b []byte + b, err = transaction.MakeSCRestAPICall( + transaction.STORAGE_CONTRACT_ADDRESS, + "/allocation", + map[string]string{"allocation": allocID}) + if err != nil { + return + } + sa := new(transaction.StorageAllocation) + err = json.Unmarshal(b, sa) + if err != nil { + return + } + allocation = &Allocation{ + ID: sa.ID, + DataShards: sa.DataShards, + ParityShards: sa.ParityShards, + Size: sa.Size, + Expiration: sa.Expiration, + Owner: sa.OwnerID, + OwnerPublicKey: sa.OwnerPublicKey, + OwnerSigningPublicKey: sa.OwnerSigningPublicKey, } + return } func NewChallengeRequest(r *http.Request) (*ChallengeRequest, string, error) { diff --git a/code/go/0chain.net/validatorcore/storage/models.go b/code/go/0chain.net/validatorcore/storage/models.go index f8b6af7b3..7ec1031b5 100644 --- a/code/go/0chain.net/validatorcore/storage/models.go +++ b/code/go/0chain.net/validatorcore/storage/models.go @@ -59,6 +59,7 @@ type RefMeta struct { FixedMerkleRoot string `json:"fixed_merkle_root"` Size int64 `json:"size"` FileMetaHash string `json:"file_meta_hash"` + SignatureVersion int `json:"signature_version"` } func (rm *RefMeta) GetFileMetaHashData(allocationID string) string { @@ -285,14 +286,15 @@ func (op *ObjectPath) Verify(allocationID string, challengeRand int64) error { } type Allocation struct { - ID string `json:"id"` - DataShards int `json:"data_shards"` - ParityShards int `json:"parity_shards"` - Size int64 `json:"size"` - UsedSize int64 `json:"used_size"` - Expiration common.Timestamp `json:"expiration_date"` - Owner string `json:"owner_id"` - OwnerPublicKey string `json:"owner_public_key"` + ID string `json:"id"` + DataShards int64 `json:"data_shards"` + ParityShards int64 `json:"parity_shards"` + Size int64 `json:"size"` + UsedSize int64 `json:"used_size"` + Expiration common.Timestamp `json:"expiration_date"` + Owner string `json:"owner_id"` + OwnerPublicKey string `json:"owner_public_key"` + OwnerSigningPublicKey string `json:"owner_signing_public_key"` } type ChallengeProof struct { @@ -322,7 +324,7 @@ func (cr *ChallengeRequest) verifyBlockNum(challengeObj *Challenge) error { } func (cr *ChallengeRequest) VerifyChallenge(challengeObj *Challenge, allocationObj *Allocation) error { - logging.Logger.Info("Verifying object path", zap.String("challenge_id", challengeObj.ID), zap.Int64("seed", challengeObj.RandomNumber), zap.Int("storage_version", cr.StorageVersion)) + logging.Logger.Info("Verifying object path", zap.String("challenge_id", challengeObj.ID), zap.Int64("seed", challengeObj.RandomNumber), zap.Int("storage_version", cr.StorageVersion), zap.String("owner_public_key", allocationObj.OwnerPublicKey)) if cr.ObjPath != nil && cr.StorageVersion == 0 { err := cr.ObjPath.Verify(challengeObj.AllocationID, challengeObj.RandomNumber) if err != nil { @@ -343,7 +345,7 @@ func (cr *ChallengeRequest) VerifyChallenge(challengeObj *Challenge, allocationO return common.NewError("write_marker_validation_failed", "Write marker timestamp does not match with challenge timestamp") } for i := 1; i < len(cr.WriteMarkers); i++ { - err = cr.WriteMarkers[i].WM.Verify(allocationObj.ID, cr.WriteMarkers[i].WM.AllocationRoot, cr.WriteMarkers[i].ClientPublicKey) + err = cr.WriteMarkers[i].WM.Verify(allocationObj.ID, cr.WriteMarkers[i].WM.AllocationRoot, allocationObj.OwnerPublicKey) if err != nil { return err } @@ -359,7 +361,7 @@ func (cr *ChallengeRequest) VerifyChallenge(challengeObj *Challenge, allocationO if len(cr.ObjectProof) == 0 && latestWM.ChainSize == 0 { return nil } - err = cr.verifyObjectProof(latestWM, challengeObj.BlobberID, cr.WriteMarkers[len(cr.WriteMarkers)-1].ClientPublicKey, challengeObj.RandomNumber) + err = cr.verifyObjectProof(latestWM, challengeObj.BlobberID, allocationObj.OwnerPublicKey, allocationObj.OwnerSigningPublicKey, challengeObj.RandomNumber) if err != nil { logging.Logger.Error("Failed to verify object proof", zap.String("challenge_id", challengeObj.ID), zap.Error(err)) return err @@ -445,7 +447,7 @@ func (vt *ValidationTicket) Sign() error { return err } -func (cr *ChallengeRequest) verifyObjectProof(latestWM *writemarker.WriteMarker, blobberID, ownerPublicKey string, challengeRand int64) error { +func (cr *ChallengeRequest) verifyObjectProof(latestWM *writemarker.WriteMarker, blobberID, ownerPublicKey, ownerSigningPublicKey string, challengeRand int64) error { if len(cr.ObjectProof) == 0 { return common.NewError("invalid_object_proof", "Object proof is missing") } @@ -481,7 +483,12 @@ func (cr *ChallengeRequest) verifyObjectProof(latestWM *writemarker.WriteMarker, // verify fixed merkle root hashData := fmt.Sprintf("%s:%s:%s:%s", cr.Meta.ActualFileHash, cr.Meta.ValidationRoot, cr.Meta.FixedMerkleRoot, blobberID) validationRootHash := encryption.Hash(hashData) - verify, err := encryption.Verify(ownerPublicKey, cr.Meta.ValidationRootSignature, validationRootHash) + var verify bool + if cr.Meta.SignatureVersion == 1 { + verify, err = encryption.VerifyEd25519(ownerSigningPublicKey, cr.Meta.ValidationRootSignature, validationRootHash) + } else { + verify, err = encryption.Verify(ownerPublicKey, cr.Meta.ValidationRootSignature, validationRootHash) + } if err != nil { logging.Logger.Error("Failed to verify the validation root signature", zap.Error(err), zap.String("validation_root", cr.Meta.ValidationRoot), zap.String("validation_root_signature", cr.Meta.ValidationRootSignature), zap.String("owner_public_key", ownerPublicKey)) return common.NewError("invalid_object_proof", "Failed to verify the validation root signature. "+err.Error()) diff --git a/code/go/0chain.net/validatorcore/storage/models_test.go b/code/go/0chain.net/validatorcore/storage/models_test.go index 221053cdb..149bd1928 100644 --- a/code/go/0chain.net/validatorcore/storage/models_test.go +++ b/code/go/0chain.net/validatorcore/storage/models_test.go @@ -572,6 +572,7 @@ func TestChallengeRequest_VerifyChallenge(t *testing.T) { RandomNumber: int64(1), AllocationID: "2", }, + alloc: &storage.Allocation{}, wantErr: true, wantErrMsg: "Invalid write marker", }, @@ -612,6 +613,7 @@ func TestChallengeRequest_VerifyChallenge(t *testing.T) { RandomNumber: int64(1), AllocationID: "1", }, + alloc: &storage.Allocation{}, wantErr: true, wantErrMsg: "Invalid write marker", }, diff --git a/goose/migrations/1730098482_session_key.sql b/goose/migrations/1730098482_session_key.sql new file mode 100644 index 000000000..9d6f76bd7 --- /dev/null +++ b/goose/migrations/1730098482_session_key.sql @@ -0,0 +1,7 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE allocations ADD COLUMN owner_signing_public_key character varying(512); + +ALTER TABLE reference_objects ADD COLUMN signature_version smallint; + +-- +goose StatementEnd \ No newline at end of file From a23f433cba1334e3faff1d172f6a04f3078b7bdb Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 30 Oct 2024 14:48:44 +0530 Subject: [PATCH 02/13] add logs for signing key --- .../blobbercore/allocation/protocol.go | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index 386a05485..9e5ba757e 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -13,6 +13,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" + "go.uber.org/zap" "gorm.io/gorm" ) @@ -112,6 +113,7 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc a.StartTime = sa.StartTime a.StorageVersion = uint8(sa.StorageVersion) a.OwnerSigningPublicKey = sa.OwnerSigningPublicKey + logging.Logger.Info("OwnerSigningPublicKey", zap.String("OwnerSigningPublicKey", a.OwnerSigningPublicKey)) m := map[string]interface{}{ "allocation_id": a.ID, @@ -145,17 +147,18 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc err = Repo.Save(ctx, a) } else { updateMap := map[string]interface{}{ - "tx": a.Tx, - "expiration_date": a.Expiration, - "owner_id": a.OwnerID, - "owner_public_key": a.OwnerPublicKey, - "repairer_id": a.RepairerID, - "size": a.TotalSize, - "finalized": a.Finalized, - "time_unit": a.TimeUnit, - "file_options": a.FileOptions, - "start_time": a.StartTime, - "blobber_size": a.BlobberSize, + "tx": a.Tx, + "expiration_date": a.Expiration, + "owner_id": a.OwnerID, + "owner_public_key": a.OwnerPublicKey, + "repairer_id": a.RepairerID, + "size": a.TotalSize, + "finalized": a.Finalized, + "time_unit": a.TimeUnit, + "file_options": a.FileOptions, + "start_time": a.StartTime, + "blobber_size": a.BlobberSize, + "owner_signing_public_key": a.OwnerSigningPublicKey, } updateOption := func(alloc *Allocation) { @@ -170,6 +173,7 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc alloc.FileOptions = a.FileOptions alloc.StartTime = a.StartTime alloc.BlobberSize = a.BlobberSize + alloc.OwnerSigningPublicKey = a.OwnerSigningPublicKey } err = Repo.UpdateAllocation(ctx, a, updateMap, updateOption) } From 142b9e76af4319e91f986b6c46a8fde5d2aa3ac2 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 30 Oct 2024 16:19:51 +0530 Subject: [PATCH 03/13] log ref --- code/go/0chain.net/blobbercore/reference/dbCollector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go index 1501d3899..0402d2ba3 100644 --- a/code/go/0chain.net/blobbercore/reference/dbCollector.go +++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go @@ -85,7 +85,7 @@ func (dc *dbCollector) Finalize(ctx context.Context, allocationID, allocationRoo err := db.Create(&(dc.createdRefs)).Error if err != nil { for ind, ref := range dc.createdRefs { - logging.Logger.Error("create_ref_error", zap.String("lookup_hash", ref.LookupHash), zap.String("path", ref.Path), zap.Int("index", ind), zap.String("allocation_root", allocationRoot)) + logging.Logger.Error("create_ref_error", zap.String("lookup_hash", ref.LookupHash), zap.String("path", ref.Path), zap.Int("index", ind), zap.String("allocation_root", allocationRoot), zap.Any("ref", ref)) } return err } From 781c0040fad263c167dfb56a9b8003244f9bc4eb Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 30 Oct 2024 22:41:02 +0530 Subject: [PATCH 04/13] increase signature length to 128 --- code/go/0chain.net/blobbercore/allocation/protocol.go | 2 +- goose/migrations/1730098482_session_key.sql | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index 9e5ba757e..81a29e2fe 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -141,7 +141,7 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc return a, nil } - logging.Logger.Info("Saving the allocation to DB") + logging.Logger.Info("Saving the allocation to DB", zap.String("allocation_id", a.ID)) if !isExist { err = Repo.Save(ctx, a) diff --git a/goose/migrations/1730098482_session_key.sql b/goose/migrations/1730098482_session_key.sql index 9d6f76bd7..9fec42ea4 100644 --- a/goose/migrations/1730098482_session_key.sql +++ b/goose/migrations/1730098482_session_key.sql @@ -4,4 +4,8 @@ ALTER TABLE allocations ADD COLUMN owner_signing_public_key character varying(51 ALTER TABLE reference_objects ADD COLUMN signature_version smallint; +ALTER TABLE reference_objects ALTER COLUMN actual_file_hash_signature TYPE character varying(128); + +ALTER TABLE reference_objects ALTER COLUMN validation_root_signature TYPE character varying(128); + -- +goose StatementEnd \ No newline at end of file From 98a8a724b6c4911411a6d1d4e59961acc01d4b76 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 31 Oct 2024 12:34:26 +0530 Subject: [PATCH 05/13] add log for validation root sig --- code/go/0chain.net/blobbercore/handler/file_command_upload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index 279e67701..0e1508c05 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -178,7 +178,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj } if err != nil || !verify { logging.Logger.Error("UploadFileCommand.VerifySignature", zap.Error(err)) - return result, common.NewError("upload_error", "Failed to verify validation root signature. ") + return result, common.NewError("upload_error", fmt.Sprintf("%s %d", "Failed to verify validation root signature ", cmd.fileChanger.SignatureVersion)) } } From 751b9a3ddeddb45c677d7e6492c07fb4c78c52f1 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 31 Oct 2024 12:35:04 +0530 Subject: [PATCH 06/13] add log for validation root sig --- code/go/0chain.net/blobbercore/handler/file_command_upload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index 0e1508c05..3969f5fda 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -177,7 +177,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj verify, err = encryption.Verify(allocationObj.OwnerPublicKey, cmd.fileChanger.ValidationRootSignature, hash) } if err != nil || !verify { - logging.Logger.Error("UploadFileCommand.VerifySignature", zap.Error(err)) + logging.Logger.Error("UploadFileCommand.VerifySignature", zap.Error(err), zap.Int("SignatureVersion", cmd.fileChanger.SignatureVersion), zap.String("Hash", hash), zap.String("ValidationRootSignature", cmd.fileChanger.ValidationRootSignature), zap.String("OwnerSigningPublicKey", allocationObj.OwnerSigningPublicKey), zap.String("OwnerPublicKey", allocationObj.OwnerPublicKey)) return result, common.NewError("upload_error", fmt.Sprintf("%s %d", "Failed to verify validation root signature ", cmd.fileChanger.SignatureVersion)) } } From 29b45231f33085829cb78df2b9729df410900b4c Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 31 Oct 2024 21:21:34 +0530 Subject: [PATCH 07/13] update common and gosdk --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 2f50da352..39bd4d399 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.22.5 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.18.0-RC8 + github.com/0chain/gosdk v1.18.0-RC8.0.20241031153959-3992e5a5e1ac github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-openapi/runtime v0.26.0 @@ -101,7 +101,7 @@ require ( ) require ( - github.com/0chain/common v1.18.1 + github.com/0chain/common v1.18.2 github.com/Luzifer/go-openssl/v3 v3.1.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/btcsuite/btcd v0.23.4 // indirect diff --git a/go.sum b/go.sum index 8a670b9e3..89bb5f5f1 100644 --- a/go.sum +++ b/go.sum @@ -36,12 +36,12 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/0chain/common v1.18.1 h1:QKXdEjK6SB3SqzDS/XbK74jy20d3tc+6PA8J8dFpuGw= -github.com/0chain/common v1.18.1/go.mod h1:Lapu2Tj7z5Sm4r+X141e7vsz4NDODTEypeElYAP3iSw= +github.com/0chain/common v1.18.2 h1:VGWfd3Xqio9xbmebPFnUbuk5QN0pK0xzvifaUggJF5g= +github.com/0chain/common v1.18.2/go.mod h1:Lapu2Tj7z5Sm4r+X141e7vsz4NDODTEypeElYAP3iSw= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.18.0-RC8 h1:QT9AVC/kDaq9dhjmKBt5ZqmhHa/LoUlSCTKAIsQSrAw= -github.com/0chain/gosdk v1.18.0-RC8/go.mod h1:fw3TUzF7HWNBuksA2WZzLcHlMz+A1XA9as5i3KUBqXg= +github.com/0chain/gosdk v1.18.0-RC8.0.20241031153959-3992e5a5e1ac h1:SxSR4rRle9iYGNwj5aznZ2CYAj7x18c5mBcfCENdj+o= +github.com/0chain/gosdk v1.18.0-RC8.0.20241031153959-3992e5a5e1ac/go.mod h1:q/zFTOMHU2hFGjFzIxCOZLonsmrSzYVP3ExCHHOmL6w= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= From 406e4ea38dd8126c947e452a5ac6e9efe4d6f084 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 31 Oct 2024 22:29:42 +0530 Subject: [PATCH 08/13] remove lock --- .../handler/object_operation_handler.go | 51 +++++++++---------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index b23a97eb9..f266dcd7c 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -1972,38 +1972,33 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob writemarkerEntity.ClientPublicKey = clientKey Logger.Info("rollback_writemarker", zap.Any("writemarker", writemarkerEntity.WM)) - alloc, err := allocation.Repo.GetByIdAndLock(c, allocationID) - Logger.Info("[rollback]Lock Allocation", zap.Bool("is_redeem_required", alloc.IsRedeemRequired), zap.String("allocation_root", alloc.AllocationRoot), zap.String("latest_wm_redeemed", alloc.LatestRedeemedWM)) - if err != nil { - txn.Rollback() - return &result, common.NewError("allocation_read_error", "Error reading the allocation object") - } + Logger.Info("[rollback]Lock Allocation", zap.Bool("is_redeem_required", allocationObj.IsRedeemRequired), zap.String("allocation_root", allocationObj.AllocationRoot), zap.String("latest_wm_redeemed", allocationObj.LatestRedeemedWM)) - alloc.BlobberSizeUsed -= latestWriteMarkerEntity.WM.Size - alloc.UsedSize -= latestWriteMarkerEntity.WM.Size - alloc.AllocationRoot = allocationRoot - alloc.FileMetaRoot = fileMetaRoot - alloc.IsRedeemRequired = false - alloc.NumObjects = alloc.PrevNumObjects - alloc.NumBlocks = alloc.PrevNumBlocks + allocationObj.BlobberSizeUsed -= latestWriteMarkerEntity.WM.Size + allocationObj.UsedSize -= latestWriteMarkerEntity.WM.Size + allocationObj.AllocationRoot = allocationRoot + allocationObj.FileMetaRoot = fileMetaRoot + allocationObj.IsRedeemRequired = false + allocationObj.NumObjects = allocationObj.PrevNumObjects + allocationObj.NumBlocks = allocationObj.PrevNumBlocks updateMap := map[string]interface{}{ - "blobber_size_used": alloc.BlobberSizeUsed, - "used_size": alloc.UsedSize, - "allocation_root": alloc.AllocationRoot, - "file_meta_root": alloc.FileMetaRoot, + "blobber_size_used": allocationObj.BlobberSizeUsed, + "used_size": allocationObj.UsedSize, + "allocation_root": allocationObj.AllocationRoot, + "file_meta_root": allocationObj.FileMetaRoot, "is_redeem_required": false, - "num_objects": alloc.NumObjects, - "num_blocks": alloc.NumBlocks, + "num_objects": allocationObj.NumObjects, + "num_blocks": allocationObj.NumBlocks, } updateOption := func(a *allocation.Allocation) { - a.BlobberSizeUsed = alloc.BlobberSizeUsed - a.UsedSize = alloc.UsedSize - a.AllocationRoot = alloc.AllocationRoot - a.FileMetaRoot = alloc.FileMetaRoot - a.IsRedeemRequired = alloc.IsRedeemRequired - a.NumObjects = alloc.NumObjects - a.NumBlocks = alloc.NumBlocks + a.BlobberSizeUsed = allocationObj.BlobberSizeUsed + a.UsedSize = allocationObj.UsedSize + a.AllocationRoot = allocationObj.AllocationRoot + a.FileMetaRoot = allocationObj.FileMetaRoot + a.IsRedeemRequired = allocationObj.IsRedeemRequired + a.NumObjects = allocationObj.NumObjects + a.NumBlocks = allocationObj.NumBlocks } writemarkerEntity.Latest = true err = txn.Create(writemarkerEntity).Error @@ -2011,7 +2006,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob txn.Rollback() return &result, common.NewError("write_marker_error", "Error persisting the write marker "+err.Error()) } - if err = allocation.Repo.UpdateAllocation(c, alloc, updateMap, updateOption); err != nil { + if err = allocation.Repo.UpdateAllocation(c, allocationObj, updateMap, updateOption); err != nil { txn.Rollback() return &result, common.NewError("allocation_write_error", "Error persisting the allocation object "+err.Error()) } @@ -2024,7 +2019,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob var node wmpt.Node if len(fileMetaRoot) > 0 { decodedRoot, _ := hex.DecodeString(fileMetaRoot) - node = wmpt.NewHashNode(decodedRoot, alloc.NumBlocks) + node = wmpt.NewHashNode(decodedRoot, allocationObj.NumBlocks) } trie.RollbackTrie(node) } From 72454ce93d4b6fe89f424e91eede02c5f376f2de Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 31 Oct 2024 22:39:08 +0530 Subject: [PATCH 09/13] run delete in same routine --- .../go/0chain.net/blobbercore/writemarker/worker.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/code/go/0chain.net/blobbercore/writemarker/worker.go b/code/go/0chain.net/blobbercore/writemarker/worker.go index c9df60763..85857b2d7 100644 --- a/code/go/0chain.net/blobbercore/writemarker/worker.go +++ b/code/go/0chain.net/blobbercore/writemarker/worker.go @@ -121,6 +121,9 @@ func redeemWriteMarker(md *markerData) error { shouldRollback := false start := time.Now() logging.Logger.Info("redeeming_write_marker", zap.String("allocationID", allocationID)) + allocMu := lock.GetMutex(allocation.Allocation{}.TableName(), allocationID) + allocMu.RLock() + defer allocMu.RUnlock() defer func() { if shouldRollback { if rollbackErr := db.Rollback().Error; rollbackErr != nil { @@ -130,14 +133,10 @@ func redeemWriteMarker(md *markerData) error { } } else { - go deleteMarkerData(allocationID) + deleteMarkerData(allocationID) } }() - allocMu := lock.GetMutex(allocation.Allocation{}.TableName(), allocationID) - allocMu.RLock() - defer allocMu.RUnlock() - alloc, err := allocation.Repo.GetAllocationFromDB(ctx, allocationID) if err != nil { logging.Logger.Error("Error redeeming the write marker.", zap.Any("allocation", allocationID), zap.Any("wm", allocationID), zap.Any("error", err)) @@ -150,7 +149,7 @@ func redeemWriteMarker(md *markerData) error { if alloc.Finalized { logging.Logger.Info("Allocation is finalized. Skipping redeeming the write marker.", zap.Any("allocation", allocationID)) - go deleteMarkerData(allocationID) + deleteMarkerData(allocationID) shouldRollback = true return nil } @@ -174,7 +173,7 @@ func redeemWriteMarker(md *markerData) error { if retryRedeem(err.Error()) { go tryAgain(md) } else { - go deleteMarkerData(allocationID) + deleteMarkerData(allocationID) } shouldRollback = true return err From d270ec76d308de237455f5b8a28ad332bd330eed Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 31 Oct 2024 23:29:19 +0530 Subject: [PATCH 10/13] read only in rollback --- code/go/0chain.net/blobbercore/allocation/protocol.go | 2 +- .../0chain.net/blobbercore/handler/object_operation_handler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index 81a29e2fe..b00f7f869 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -113,7 +113,7 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc a.StartTime = sa.StartTime a.StorageVersion = uint8(sa.StorageVersion) a.OwnerSigningPublicKey = sa.OwnerSigningPublicKey - logging.Logger.Info("OwnerSigningPublicKey", zap.String("OwnerSigningPublicKey", a.OwnerSigningPublicKey)) + logging.Logger.Info("OwnerSigningPublicKey", zap.String("OwnerSigningPublicKey", a.OwnerSigningPublicKey), zap.String("allocation_id", a.ID)) m := map[string]interface{}{ "allocation_id": a.ID, diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index f266dcd7c..5d17ca0c4 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -1832,7 +1832,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob err error ) - allocationObj, err = fsh.verifyAllocation(ctx, allocationId, allocationTx, false) + allocationObj, err = fsh.verifyAllocation(ctx, allocationId, allocationTx, true) if err != nil { Logger.Error("Error in verifying allocation", zap.Error(err)) return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) From 53d64f58e84c287707a59228cfa65828720cb1b0 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 1 Nov 2024 03:01:55 +0530 Subject: [PATCH 11/13] allow writes in get latest marker --- code/go/0chain.net/blobbercore/handler/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index 963ec12f0..ccd9afb73 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -207,7 +207,7 @@ func setupHandlers(s *mux.Router) { RateLimitByObjectRL(common.ToJSONResponse(WithReadOnlyConnection(ReferencePathV2Handler)))) s.HandleFunc("/v1/file/latestwritemarker/{allocation}", - RateLimitByObjectRL(common.ToJSONResponse(WithReadOnlyConnection(WriteMarkerHandler)))) + RateLimitByObjectRL(common.ToJSONResponse(WithConnection(WriteMarkerHandler)))) s.HandleFunc("/v1/file/objecttree/{allocation}", RateLimitByObjectRL(common.ToStatusCode(WithStatusReadOnlyConnection(ObjectTreeHandler)))). From 6e2bb49b5d85dab64fb02ed4a446e586d0d8349c Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 1 Nov 2024 13:39:21 +0530 Subject: [PATCH 12/13] fix config for max files --- code/go/0chain.net/blobbercore/config/config.go | 2 +- config/0chain_blobber.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index 62adce13a..3d5e457a1 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -53,7 +53,7 @@ func SetupDefaultConfig() { viper.SetDefault("max_dirs_files", 50000) viper.SetDefault("max_objects_dir", 1000) - viper.SetDefault("max_objects_per_gb", 100000) + viper.SetDefault("max_objects_per_gb", 1000) viper.SetDefault("kv.pebble_dir", "/pebble/data") viper.SetDefault("kv.pebble_wal_dir", "/pebble/wal") viper.SetDefault("kv.pebble_cache", 4*1024*1024*1024) diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml index 55b5bcf37..c7864e651 100755 --- a/config/0chain_blobber.yaml +++ b/config/0chain_blobber.yaml @@ -29,6 +29,8 @@ max_dirs_files: 50000 # maximum limit on the number of objects in a directory max_objects_dir: 1000 +# limit of objects per gb(storage v2) +max_objects_per_gb: 1000 # delegate wallet (must be set) delegate_wallet: '9c693cb14f29917968d6e8c909ebbea3425b4c1bc64b6732cadc2a1869f49be9' From 84b7d56824d3d16ff18f3f92a79b440d328fe093 Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Fri, 1 Nov 2024 14:33:19 +0530 Subject: [PATCH 13/13] Update gosdk --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 39bd4d399..79c8f351a 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.22.5 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.18.0-RC8.0.20241031153959-3992e5a5e1ac + github.com/0chain/gosdk v1.18.0-RC8.0.20241101062307-6d2b9806e139 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-openapi/runtime v0.26.0 diff --git a/go.sum b/go.sum index 89bb5f5f1..6e3b3ee05 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/0chain/common v1.18.2 h1:VGWfd3Xqio9xbmebPFnUbuk5QN0pK0xzvifaUggJF5g= github.com/0chain/common v1.18.2/go.mod h1:Lapu2Tj7z5Sm4r+X141e7vsz4NDODTEypeElYAP3iSw= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.18.0-RC8.0.20241031153959-3992e5a5e1ac h1:SxSR4rRle9iYGNwj5aznZ2CYAj7x18c5mBcfCENdj+o= -github.com/0chain/gosdk v1.18.0-RC8.0.20241031153959-3992e5a5e1ac/go.mod h1:q/zFTOMHU2hFGjFzIxCOZLonsmrSzYVP3ExCHHOmL6w= +github.com/0chain/gosdk v1.18.0-RC8.0.20241101062307-6d2b9806e139 h1:ZYPT+ym1f+Rx2OXvS2na5De9FY/JtDXtbMfxVua3ukk= +github.com/0chain/gosdk v1.18.0-RC8.0.20241101062307-6d2b9806e139/go.mod h1:q/zFTOMHU2hFGjFzIxCOZLonsmrSzYVP3ExCHHOmL6w= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=