diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 30d0aef92..9d78f2677 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -16,6 +16,8 @@ env: BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }} VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} DOCKER_CLI_EXPERIMENTAL: enabled + BLOBBER_BUILDBASE: blobber_base + BLOBBER_BUILD_BASE_REGISTRY: ${{ secrets.BLOBBER_BUILD_BASE_REGISTRY }} jobs: blobber: @@ -39,7 +41,7 @@ jobs: go-version: ^1.20 # The Go version to download (if necessary) and use. - name: Clone blobber - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: Set up Docker Buildx run: | @@ -60,6 +62,26 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Get changed files using defaults + id: changed-files + uses: tj-actions/changed-files@v18.4 + + - name: Pull Build Base + run: | + docker pull $BLOBBER_BUILD_BASE_REGISTRY:staging + docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE + + - name: Build Base image + if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile') + run: | + SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) + + ./docker.local/bin/build.base.sh && + docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG + docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA + docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG + docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA + - name: Build blobber run: | SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) @@ -68,7 +90,7 @@ jobs: export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:${TAG} -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}" docker buildx create --driver-opt network=host --use --buildkitd-flags '--allow-insecure-entitlement security.insecure' --use blobber_buildx - ./docker.local/bin/build.base.sh && ./docker.local/bin/build.blobber.sh + ./docker.local/bin/build.blobber.sh validator: runs-on: [self-hosted, arc-runner] @@ -112,6 +134,26 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Get changed files using defaults + id: changed-files + uses: tj-actions/changed-files@v18.4 + + - name: Pull Build Base + run: | + docker pull $BLOBBER_BUILD_BASE_REGISTRY:staging + docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE + + - name: Build Base image + if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile') + run: | + SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) + + ./docker.local/bin/build.base.sh + docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG + docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA + docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG + docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA + - name: Build validator run: | SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) @@ -119,7 +161,7 @@ jobs: export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" export DOCKER_IMAGE_VALIDATOR="-t ${VALIDATOR_REGISTRY}:${TAG} -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}" docker buildx create --driver-opt network=host --use --buildkitd-flags '--allow-insecure-entitlement security.insecure' --use blobber_buildx - ./docker.local/bin/build.base.sh && ./docker.local/bin/build.validator.sh + ./docker.local/bin/build.validator.sh system-tests: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e96dfd18..7991a985c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,10 +6,7 @@ concurrency: on: push: - branches: - - master - - staging - tags: + branches: [ master, staging, sprint* ] pull_request: jobs: diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go index ce8219059..96b6da436 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go @@ -375,9 +375,7 @@ func TestBlobberCore_CopyFile(t *testing.T) { config.Configuration.MaxAllocationDirFiles = tc.maxDirFilesPerAlloc - ctx := context.TODO() - db := datastore.GetStore().GetDB().Begin() - ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + ctx := datastore.GetStore().CreateTransaction(context.TODO()) change := &CopyFileChange{ AllocationID: tc.allocationID, diff --git a/code/go/0chain.net/blobbercore/allocation/dao.go b/code/go/0chain.net/blobbercore/allocation/dao.go index d7b734564..edeea5ed1 100644 --- a/code/go/0chain.net/blobbercore/allocation/dao.go +++ b/code/go/0chain.net/blobbercore/allocation/dao.go @@ -2,6 +2,7 @@ package allocation import ( "context" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/errors" @@ -12,33 +13,25 @@ import ( // GetOrCreate, get allocation if it exists in db. if not, try to sync it from blockchain, and insert it in db. func GetOrCreate(ctx context.Context, store datastore.Store, allocationId string) (*Allocation, error) { - db := store.GetDB() + db := store.CreateTransaction(ctx) if len(allocationId) == 0 { return nil, errors.Throw(constants.ErrInvalidParameter, "tx") } - alloc := &Allocation{} - result := db.Table(TableNameAllocation).Where(SQLWhereGetById, allocationId).First(alloc) - - if result.Error == nil { + alloc, err := Repo.GetById(db, allocationId) + if err == nil { return alloc, nil } + if !errors.Is(err, gorm.ErrRecordNotFound) { - if !errors.Is(result.Error, gorm.ErrRecordNotFound) { - - return nil, errors.ThrowLog(result.Error.Error(), common.ErrBadDataStore) + return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } return SyncAllocation(allocationId) } -const ( - SQLWhereGetByTx = "allocations.tx = ?" - SQLWhereGetById = "allocations.id = ?" -) - // DryRun Creates a prepared statement when executing any SQL and caches them to speed up future calls // https://gorm.io/docs/performance.html#Caches-Prepared-Statement func DryRun(db *gorm.DB) { diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go index 5761f889c..1688e4b75 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go @@ -254,9 +254,7 @@ func TestBlobberCore_DeleteFile(t *testing.T) { config.Configuration.MaxAllocationDirFiles = tc.maxDirFilesPerAlloc - ctx := context.TODO() - db := datastore.GetStore().GetDB().Begin() - ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + ctx := datastore.GetStore().CreateTransaction(context.TODO()) change := &DeleteFileChange{ AllocationID: tc.allocationID, diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go index d4d953d98..4530a8322 100644 --- a/code/go/0chain.net/blobbercore/allocation/entity.go +++ b/code/go/0chain.net/blobbercore/allocation/entity.go @@ -1,10 +1,12 @@ package allocation import ( + "context" "errors" "fmt" "time" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "gorm.io/gorm/clause" @@ -162,7 +164,8 @@ func (*Pending) TableName() string { } // GetPendingWrite Get write size that is not yet redeemed -func GetPendingWrite(db *gorm.DB, clientID, allocationID string) (pendingWriteSize int64, err error) { +func GetPendingWrite(ctx context.Context, clientID, allocationID string) (pendingWriteSize int64, err error) { + db := datastore.GetStore().GetTransaction(ctx) err = db.Model(&Pending{}).Select("pending_write").Where( "id=?", fmt.Sprintf("%v:%v", clientID, allocationID), ).Scan(&pendingWriteSize).Error @@ -177,7 +180,8 @@ func GetPendingWrite(db *gorm.DB, clientID, allocationID string) (pendingWriteSi } // GetPendingRead Get read size that is not yet redeemed -func GetPendingRead(db *gorm.DB, clientID, allocationID string) (pendingReadSize int64, err error) { +func GetPendingRead(ctx context.Context, clientID, allocationID string) (pendingReadSize int64, err error) { + db := datastore.GetStore().GetTransaction(ctx) err = db.Model(&Pending{}).Select("pending_read").Where( "id=?", fmt.Sprintf("%v:%v", clientID, allocationID), ).Scan(&pendingReadSize).Error @@ -191,7 +195,8 @@ func GetPendingRead(db *gorm.DB, clientID, allocationID string) (pendingReadSize return } -func AddToPending(db *gorm.DB, clientID, allocationID string, pendingWrite int64) (err error) { +func AddToPending(ctx context.Context, clientID, allocationID string, pendingWrite int64) (err error) { + db := datastore.GetStore().GetTransaction(ctx) key := clientID + ":" + allocationID // Lock is required because two process can simultaneously call this function and read pending data // thus giving same value leading to inconsistent data @@ -215,7 +220,8 @@ func AddToPending(db *gorm.DB, clientID, allocationID string, pendingWrite int64 return nil } -func GetWritePoolsBalance(db *gorm.DB, allocationID string) (balance uint64, err error) { +func GetWritePoolsBalance(ctx context.Context, allocationID string) (balance uint64, err error) { + db := datastore.GetStore().GetTransaction(ctx) err = db.Model(&WritePool{}).Select("COALESCE(SUM(balance),0) as tot_balance").Where( "allocation_id = ?", allocationID, ).Scan(&balance).Error @@ -266,13 +272,14 @@ func (WritePool) TableName() string { return "write_pools" } -func GetReadPool(db *gorm.DB, clientID string) (*ReadPool, error) { +func GetReadPool(ctx context.Context, clientID string) (*ReadPool, error) { + db := datastore.GetStore().GetTransaction(ctx) var rp ReadPool return &rp, db.Model(&ReadPool{}).Where("client_id = ?", clientID).Scan(&rp).Error } -func GetReadPoolsBalance(db *gorm.DB, clientID string) (int64, error) { - rp, err := GetReadPool(db, clientID) +func GetReadPoolsBalance(ctx context.Context, clientID string) (int64, error) { + rp, err := GetReadPool(ctx, clientID) if err != nil { return 0, err } @@ -280,22 +287,24 @@ func GetReadPoolsBalance(db *gorm.DB, clientID string) (int64, error) { return rp.Balance, nil } -func UpsertReadPool(db *gorm.DB, rp *ReadPool) error { +func UpsertReadPool(ctx context.Context, rp *ReadPool) error { updateFields := []string{"balance"} - + db := datastore.GetStore().GetTransaction(ctx) return db.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "client_id"}}, DoUpdates: clause.AssignmentColumns(updateFields), // column needed to be updated }).Create(&rp).Error } -func UpdateReadPool(db *gorm.DB, rp *ReadPool) error { +func UpdateReadPool(ctx context.Context, rp *ReadPool) error { + db := datastore.GetStore().GetTransaction(ctx) return db.Model(&ReadPool{}).Where("client_id = ?", rp.ClientID).Updates(map[string]interface{}{ "balance": rp.Balance, }).Error } -func SetWritePool(db *gorm.DB, allocationID string, wp *WritePool) (err error) { +func SetWritePool(ctx context.Context, allocationID string, wp *WritePool) (err error) { + db := datastore.GetStore().GetTransaction(ctx) err = db.Delete(&WritePool{}, "allocation_id = ?", allocationID).Error if err != nil { return diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go index b2cd22da6..f75c8624f 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go @@ -95,9 +95,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) { config.Configuration.MaxAllocationDirFiles = tc.maxDirFilesPerAlloc - ctx := context.TODO() - db := datastore.GetStore().GetDB().Begin() - ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + ctx := datastore.GetStore().CreateTransaction(context.TODO()) fPath := "/new" change := &UploadFileChanger{ diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go index 60600cc4d..ebdc70d12 100644 --- a/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go @@ -468,9 +468,7 @@ func TestBlobberCore_MoveFile(t *testing.T) { config.Configuration.MaxAllocationDirFiles = tc.maxDirFilesPerAlloc - ctx := context.TODO() - db := datastore.GetStore().GetDB().Begin() - ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + ctx := datastore.GetStore().CreateTransaction(context.TODO()) change := &MoveFileChange{ AllocationID: tc.allocationID, diff --git a/code/go/0chain.net/blobbercore/allocation/multiop_test.go b/code/go/0chain.net/blobbercore/allocation/multiop_test.go index 3d48e23c6..917cd9030 100644 --- a/code/go/0chain.net/blobbercore/allocation/multiop_test.go +++ b/code/go/0chain.net/blobbercore/allocation/multiop_test.go @@ -32,9 +32,7 @@ func TestMultiOp(t *testing.T) { alloc.OwnerPublicKey = sch.GetPublicKey() alloc.OwnerID = client.GetClientID() datastore.MocketTheStore(t, true) - ctx := context.TODO() - db := datastore.GetStore().GetDB().Begin() - ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + ctx := datastore.GetStore().CreateTransaction(context.TODO()) setupDbMock() fileIDMeta := make(map[string]string) fileIDMeta["/"] = randName() diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index a50b3cbf8..7edd76cfa 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -14,19 +14,12 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/blobber/code/go/0chain.net/core/transaction" - "go.uber.org/zap" "gorm.io/gorm" ) // GetAllocationByID from DB. This function doesn't load related terms. func GetAllocationByID(ctx context.Context, allocID string) (a *Allocation, err error) { - var tx = datastore.GetStore().GetTransaction(ctx) - - a = new(Allocation) - err = tx.Model(&Allocation{}). - Where("id=?", allocID). - First(a).Error - return + return Repo.GetById(ctx, allocID) } // LoadTerms loads corresponding terms from DB. Since, the GetAllocationByID @@ -49,14 +42,11 @@ func (a *Allocation) LoadTerms(ctx context.Context) (err error) { return // found in DB } -// VerifyAllocationTransaction try to get allocation from postgres.if it doesn't exists, get it from sharders, and insert it into postgres. +// FetchAllocationFromEventsDB try to get allocation from postgres.if it doesn't exists, get it from sharders, and insert it into postgres. func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, allocationTx string, readonly bool) (a *Allocation, err error) { var tx = datastore.GetStore().GetTransaction(ctx) - a = new(Allocation) - err = tx.Model(&Allocation{}). - Where(&Allocation{Tx: allocationTx}). - First(a).Error + a, err = Repo.GetByTx(ctx, allocationID, allocationTx) if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { return nil, common.NewError("bad_db_operation", err.Error()) // unexpected DB error @@ -81,21 +71,13 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc } var isExist bool - err = tx.Model(&Allocation{}). - Where("id = ?", sa.ID). - First(a).Error + a, err = Repo.GetById(ctx, sa.ID) if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { return nil, common.NewError("bad_db_operation", err.Error()) // unexpected } isExist = a.ID != "" - logging.Logger.Info("VerifyAllocationTransaction", - zap.Bool("isExist", isExist), - zap.Any("allocation", a), - zap.Any("storageAllocation", sa), - zap.String("node.Self.ID", node.Self.ID)) - if !isExist { foundBlobber := false for _, blobberConnection := range sa.BlobberDetails { @@ -136,6 +118,7 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc if err != nil { return nil, common.NewError("meta_data_update_error", err.Error()) } + // go update allocation data in file store map // related terms a.Terms = make([]*Terms, 0, len(sa.BlobberDetails)) @@ -154,7 +137,8 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc logging.Logger.Info("Saving the allocation to DB") - err = tx.Save(a).Error + err = Repo.Save(ctx, a) + if err != nil { return nil, err } @@ -181,13 +165,16 @@ func RequestReadPoolStat(clientID string) (*ReadPool, error) { return nil, fmt.Errorf("requesting read pools stat: %v", err) } - var readPool ReadPool - if err = json.Unmarshal(resp, &readPool); err != nil { - return nil, fmt.Errorf("decoding read pools stat response: %v, \n==> resp: %s", err, string(resp)) + if resp != nil { + var readPool ReadPool + if err = json.Unmarshal(resp, &readPool); err != nil { + return nil, fmt.Errorf("decoding read pools stat response: %v, \n==> resp: %s", err, string(resp)) + } + readPool.ClientID = clientID + return &readPool, nil } - readPool.ClientID = clientID - return &readPool, nil + return nil, errors.New("empty response received from MakeSCRestAPICall") } func RequestWritePool(allocationID string) (wps *WritePool, err error) { @@ -205,16 +192,20 @@ func RequestWritePool(allocationID string) (wps *WritePool, err error) { return nil, fmt.Errorf("requesting write pools stat: %v", err) } - var allocation = struct { - ID string `json:"id"` - WritePool uint64 `json:"write_pool"` - }{} - if err = json.Unmarshal(resp, &allocation); err != nil { - return nil, fmt.Errorf("decoding write pools stat response: %v", err) + if resp != nil { + var allocation = struct { + ID string `json:"id"` + WritePool uint64 `json:"write_pool"` + }{} + if err = json.Unmarshal(resp, &allocation); err != nil { + return nil, fmt.Errorf("decoding write pools stat response: %v", err) + } + + return &WritePool{ + AllocationID: allocationID, + Balance: allocation.WritePool, + }, nil } - return &WritePool{ - AllocationID: allocationID, - Balance: allocation.WritePool, - }, nil + return nil, errors.New("empty response received from MakeSCRestAPICall") } diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go index b65e7c587..2be71a223 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_test.go @@ -431,9 +431,7 @@ func TestBlobberCore_RenameFile(t *testing.T) { datastore.MocketTheStore(t, true) tc.setupDbMock() - ctx := context.TODO() - db := datastore.GetStore().GetDB().Begin() - ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + ctx := datastore.GetStore().CreateTransaction(context.TODO()) t.Run(tc.name, func(t *testing.T) { change := &RenameFileChange{AllocationID: alloc.ID, Path: tc.path, NewName: tc.newName} rootRef, err := reference.GetReferencePathFromPaths(ctx, alloc.ID, []string{change.Path}, []string{}) diff --git a/code/go/0chain.net/blobbercore/allocation/repository.go b/code/go/0chain.net/blobbercore/allocation/repository.go new file mode 100644 index 000000000..e8bd9e7b9 --- /dev/null +++ b/code/go/0chain.net/blobbercore/allocation/repository.go @@ -0,0 +1,187 @@ +package allocation + +import ( + "context" + "fmt" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +const ( + SQLWhereGetById = "allocations.id = ?" + SQLWhereGetByTx = "allocations.tx = ?" +) + +var ( + Repo *Repository +) + +func init() { + Repo = &Repository{} +} + +type Repository struct { +} + +type Res struct { + ID string +} + +func (r *Repository) GetById(ctx context.Context, id string) (*Allocation, error) { + tx := datastore.GetStore().GetTransaction(ctx) + if tx == nil { + logging.Logger.Panic("no transaction in the context") + } + + cache, err := getCache(tx) + if err != nil { + return nil, err + } + + if a, ok := cache[id]; ok { + return a, nil + } + + alloc := &Allocation{} + err = tx.Table(TableNameAllocation).Where(SQLWhereGetById, id).First(alloc).Error + if err != nil { + return alloc, err + } + + cache[id] = alloc + + return alloc, nil +} + +func (r *Repository) GetByIdAndLock(ctx context.Context, id string) (*Allocation, error) { + var tx = datastore.GetStore().GetTransaction(ctx) + if tx == nil { + logging.Logger.Panic("no transaction in the context") + } + + cache, err := getCache(tx) + if err != nil { + return nil, err + } + + alloc := &Allocation{} + + err = tx.Model(&Allocation{}). + Clauses(clause.Locking{Strength: "NO KEY UPDATE"}). + Where("id=?", id). + First(alloc).Error + if err != nil { + return alloc, err + } + cache[id] = alloc + + return alloc, err +} + +func (r *Repository) GetByTx(ctx context.Context, allocationID, txHash string) (*Allocation, error) { + var tx = datastore.GetStore().GetTransaction(ctx) + if tx == nil { + logging.Logger.Panic("no transaction in the context") + } + + cache, err := getCache(tx) + if err != nil { + return nil, err + } + if a, ok := cache[allocationID]; ok { + if a.Tx == txHash { + return a, nil + } + } + + alloc := &Allocation{} + err = tx.Table(TableNameAllocation).Where(SQLWhereGetByTx, txHash).First(alloc).Error + if err != nil { + return alloc, err + } + cache[allocationID] = alloc + + return alloc, err +} + +func (r *Repository) GetAllocations(ctx context.Context, offset int64) ([]*Allocation, error) { + var tx = datastore.GetStore().GetTransaction(ctx) + + const query = `finalized = false AND cleaned_up = false` + allocs := make([]*Allocation, 0) + return allocs, tx.Model(&Allocation{}). + Where(query). + Limit(UPDATE_LIMIT). + Offset(int(offset)). + Order("id ASC"). + Find(&allocs).Error +} + +func (r *Repository) GetAllocationIds(ctx context.Context) []Res { + var tx = datastore.GetStore().GetTransaction(ctx) + if tx == nil { + logging.Logger.Panic("no transaction in the context") + } + + var res []Res + + err := tx.Model(&Allocation{}).Select("id").Find(&res).Error + if err != nil && err != gorm.ErrRecordNotFound { + logging.Logger.Error("error_getting_allocations_worker", + zap.Any("error", err)) + } + + return res + +} + +func (r *Repository) UpdateAllocationRedeem(ctx context.Context, allocationID, AllocationRoot string) error { + var tx = datastore.GetStore().GetTransaction(ctx) + if tx == nil { + logging.Logger.Panic("no transaction in the context") + } + + cache, err := getCache(tx) + if err != nil { + return err + } + delete(cache, allocationID) + + err = tx.Exec("UPDATE allocations SET latest_redeemed_write_marker=?,is_redeem_required=? WHERE id=?", + AllocationRoot, false, allocationID).Error + + return err +} + +func (r *Repository) Save(ctx context.Context, a *Allocation) error { + var tx = datastore.GetStore().GetTransaction(ctx) + if tx == nil { + logging.Logger.Panic("no transaction in the context") + } + + cache, err := getCache(tx) + if err != nil { + return err + } + + cache[a.ID] = a + return tx.Save(a).Error +} + +func getCache(tx *datastore.EnhancedDB) (map[string]*Allocation, error) { + c, ok := tx.SessionCache[TableNameAllocation] + if ok { + cache, ok := c.(map[string]*Allocation) + if !ok { + return nil, fmt.Errorf("type assertion failed") + } + return cache, nil + } + cache := make(map[string]*Allocation) + tx.SessionCache[TableNameAllocation] = cache + return cache, nil +} diff --git a/code/go/0chain.net/blobbercore/allocation/rollback.go b/code/go/0chain.net/blobbercore/allocation/rollback.go index b656daee7..fe25adaac 100644 --- a/code/go/0chain.net/blobbercore/allocation/rollback.go +++ b/code/go/0chain.net/blobbercore/allocation/rollback.go @@ -6,7 +6,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "gorm.io/gorm" ) func ApplyRollback(ctx context.Context, allocationID string) error { @@ -15,24 +14,17 @@ func ApplyRollback(ctx context.Context, allocationID string) error { // delete all is_precommit rows - err := db.Transaction(func(tx *gorm.DB) error { - err := db.Model(&reference.Ref{}).Unscoped(). - Delete(&reference.Ref{}, - "allocation_id=? AND is_precommit=? AND deleted_at IS NULL", - allocationID, true).Error - if err != nil { - return err - } - - // err = db.Exec("UPDATE file_stats SET deleted_at=NULL WHERE ref_id IN (SELECT id FROM reference_objects WHERE allocation_id=? AND deleted_at IS NOT NULL)", allocationID).Error - // revive soft deleted ref rows - err = db.Exec("UPDATE reference_objects SET deleted_at=NULL,is_precommit=? WHERE allocation_id=? AND deleted_at IS NOT NULL", false, allocationID).Error - if err != nil { - return err - } - return nil - }) - + err := db.Model(&reference.Ref{}).Unscoped(). + Delete(&reference.Ref{}, + "allocation_id=? AND is_precommit=? AND deleted_at IS NULL", + allocationID, true).Error + if err != nil { + return err + } + + // err = db.Exec("UPDATE file_stats SET deleted_at=NULL WHERE ref_id IN (SELECT id FROM reference_objects WHERE allocation_id=? AND deleted_at IS NOT NULL)", allocationID).Error + // revive soft deleted ref rows + err = db.Exec("UPDATE reference_objects SET deleted_at=NULL,is_precommit=? WHERE allocation_id=? AND deleted_at IS NOT NULL", false, allocationID).Error return err } diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go index bb739277e..d6e2238e6 100644 --- a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go @@ -336,9 +336,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) { datastore.MocketTheStore(t, true) tc.setupDbMock() - ctx := context.TODO() - db := datastore.GetStore().GetDB().Begin() - ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + ctx := datastore.GetStore().CreateTransaction(context.TODO()) change := &UpdateFileChanger{ BaseFileChanger: BaseFileChanger{ diff --git a/code/go/0chain.net/blobbercore/allocation/workers.go b/code/go/0chain.net/blobbercore/allocation/workers.go index 6096a0541..952c35a7c 100644 --- a/code/go/0chain.net/blobbercore/allocation/workers.go +++ b/code/go/0chain.net/blobbercore/allocation/workers.go @@ -79,7 +79,7 @@ func updateWork(ctx context.Context) { var ( allocs []*Allocation - count int64 + count int offset int64 err error @@ -87,8 +87,8 @@ func updateWork(ctx context.Context) { // iterate all in loop accepting allocations with limit - for start := true; start || (offset < count); start = false { - allocs, count, err = FindAllocations(ctx, offset) + for start := true; start || (offset < int64(count)); start = false { + allocs, count, err = findAllocations(ctx, offset) if err != nil { logging.Logger.Error("finding allocations in DB", zap.Error(err)) if waitOrQuit(ctx, UPDATE_DB_INTERVAL) { @@ -109,28 +109,15 @@ func updateWork(ctx context.Context) { } // not finalized, not cleaned up -func FindAllocations(ctx context.Context, offset int64) (allocs []*Allocation, count int64, err error) { - const query = `finalized = false AND cleaned_up = false` +func findAllocations(ctx context.Context, offset int64) (allocs []*Allocation, count int, err error) { ctx = datastore.GetStore().CreateTransaction(ctx) var tx = datastore.GetStore().GetTransaction(ctx) defer tx.Rollback() - err = tx.Model(&Allocation{}).Where(query).Count(&count).Error - if err != nil { - logging.Logger.Error(err.Error()) - return - } - - allocs = make([]*Allocation, 0) - err = tx.Model(&Allocation{}). - Where(query). - Limit(UPDATE_LIMIT). - Offset(int(offset)). - Order("id ASC"). - Find(&allocs).Error - return + allocations, err := Repo.GetAllocations(ctx, offset) + return allocations, len(allocations), err } func shouldFinalize(sa *transaction.StorageAllocation) bool { @@ -197,7 +184,7 @@ func updateAllocationInDB(ctx context.Context, a *Allocation, sa *transaction.St ctx = datastore.GetStore().CreateTransaction(ctx) var tx = datastore.GetStore().GetTransaction(ctx) - defer commit(tx, &err) + defer commit(tx.DB, &err) var changed bool = a.Tx != sa.Tx @@ -224,7 +211,7 @@ func updateAllocationInDB(ctx context.Context, a *Allocation, sa *transaction.St } // save allocations - if err := tx.Save(a).Error; err != nil { + if err := Repo.Save(ctx, a); err != nil { return nil, err } @@ -239,6 +226,7 @@ func updateAllocationInDB(ctx context.Context, a *Allocation, sa *transaction.St } } + logging.Logger.Info("allocation updated", zap.String("id", a.ID), zap.Any("a", a)) return a, nil // ok } @@ -276,7 +264,7 @@ func cleanupAllocation(ctx context.Context, a *Allocation) { ctx = datastore.GetStore().CreateTransaction(ctx) var tx = datastore.GetStore().GetTransaction(ctx) - defer commit(tx, &err) + defer commit(tx.DB, &err) a.CleanedUp = true if err = tx.Model(a).Updates(a).Error; err != nil { @@ -287,7 +275,7 @@ func cleanupAllocation(ctx context.Context, a *Allocation) { func deleteAllocation(ctx context.Context, a *Allocation) (err error) { ctx = datastore.GetStore().CreateTransaction(ctx) var tx = datastore.GetStore().GetTransaction(ctx) - defer commit(tx, &err) + defer commit(tx.DB, &err) filestore.GetFileStore().DeleteAllocation(a.ID) err = tx.Model(&reference.Ref{}).Unscoped(). diff --git a/code/go/0chain.net/blobbercore/allocation/zcn.go b/code/go/0chain.net/blobbercore/allocation/zcn.go index 0dff85ceb..3ff8ac26b 100644 --- a/code/go/0chain.net/blobbercore/allocation/zcn.go +++ b/code/go/0chain.net/blobbercore/allocation/zcn.go @@ -1,12 +1,16 @@ package allocation import ( + "context" + "math" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/errors" + "go.uber.org/zap" "gorm.io/gorm" - "math" ) // SyncAllocation try to pull allocation from blockchain, and insert it in db. @@ -62,7 +66,8 @@ func SyncAllocation(allocationId string) (*Allocation, error) { } err = datastore.GetStore().GetDB().Transaction(func(tx *gorm.DB) error { - if err := tx.Table(TableNameAllocation).Save(alloc).Error; err != nil { + ctx := datastore.GetStore().WithTransaction(context.Background(), tx) + if err := Repo.Save(ctx, alloc); err != nil { return err } @@ -75,5 +80,15 @@ func SyncAllocation(allocationId string) (*Allocation, error) { return nil }) + if err != nil { + return nil, errors.Throw(err, "meta_data_update_error", err.Error()) + } + + logging.Logger.Info("Saving the allocation to DB", zap.Any( + "allocation", alloc), zap.Error(err)) + if err != nil { + return nil, err + } + return alloc, err } diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go index 3f77cb801..4bdea9cfb 100644 --- a/code/go/0chain.net/blobbercore/challenge/challenge.go +++ b/code/go/0chain.net/blobbercore/challenge/challenge.go @@ -24,25 +24,23 @@ type BCChallengeResponse struct { Challenges []*ChallengeEntity `json:"challenges"` } -var lastChallengeTimestamp int +var lastChallengeRound int64 func syncOpenChallenges(ctx context.Context) { - const incrOffset = 20 defer func() { if r := recover(); r != nil { logging.Logger.Error("[recover]challenge", zap.Any("err", r)) } }() - offset := 0 params := make(map[string]string) params["blobber"] = node.Self.ID - params["offset"] = strconv.Itoa(offset) - params["limit"] = "20" - if lastChallengeTimestamp > 0 { - params["from"] = strconv.Itoa(lastChallengeTimestamp) + + params["limit"] = "50" + if lastChallengeRound > 0 { + params["from"] = strconv.FormatInt(lastChallengeRound, 10) } - logging.Logger.Info("[challenge]sync:pull", zap.Any("params", params)) + start := time.Now() var downloadElapsed, jsonElapsed time.Duration @@ -54,6 +52,9 @@ func syncOpenChallenges(ctx context.Context) { return default: } + + logging.Logger.Info("[challenge]sync:pull", zap.Any("params", params)) + var challenges BCChallengeResponse var challengeIDs []string challenges.Challenges = make([]*ChallengeEntity, 0) @@ -75,13 +76,13 @@ func syncOpenChallenges(ctx context.Context) { break } sort.Slice(challenges.Challenges, func(i, j int) bool { - return challenges.Challenges[i].CreatedAt < challenges.Challenges[j].CreatedAt + return challenges.Challenges[i].RoundCreatedAt < challenges.Challenges[j].RoundCreatedAt }) count += len(challenges.Challenges) for _, c := range challenges.Challenges { challengeIDs = append(challengeIDs, c.ChallengeID) - if c.CreatedAt > common.Timestamp(lastChallengeTimestamp) { - lastChallengeTimestamp = int(c.CreatedAt) + if c.RoundCreatedAt >= lastChallengeRound { + lastChallengeRound = c.RoundCreatedAt } toProcessChallenge <- c } @@ -93,8 +94,6 @@ func syncOpenChallenges(ctx context.Context) { if len(challenges.Challenges) == 0 { break } - offset += incrOffset - params["offset"] = strconv.Itoa(offset) } dbTimeStart := time.Now() @@ -110,6 +109,11 @@ func syncOpenChallenges(ctx context.Context) { func validateOnValidators(c *ChallengeEntity) { + logging.Logger.Info("[challenge]validate: ", + zap.Any("challenge", c), + zap.String("challenge_id", c.ChallengeID), + ) + ctx := datastore.GetStore().CreateTransaction(context.TODO()) defer ctx.Done() @@ -119,7 +123,7 @@ func validateOnValidators(c *ChallengeEntity) { logging.Logger.Error("[challengetiming]add: ", zap.String("challenge_id", c.ChallengeID), zap.Error(err)) - deleteChallenge(int64(c.CreatedAt)) + deleteChallenge(c.RoundCreatedAt) tx.Rollback() } @@ -150,7 +154,7 @@ func validateOnValidators(c *ChallengeEntity) { zap.Time("created", createdTime), zap.Error(err)) //TODO: Should we delete the challenge from map or send it back to the todo channel? - deleteChallenge(int64(c.CreatedAt)) + deleteChallenge(c.RoundCreatedAt) tx.Rollback() return } diff --git a/code/go/0chain.net/blobbercore/challenge/entity.go b/code/go/0chain.net/blobbercore/challenge/entity.go index 30eb0bf39..6218673fa 100644 --- a/code/go/0chain.net/blobbercore/challenge/entity.go +++ b/code/go/0chain.net/blobbercore/challenge/entity.go @@ -97,8 +97,9 @@ type ChallengeEntity struct { Timestamp common.Timestamp `gorm:"column:timestamp;not null;default:0" json:"timestamp"` // This time is taken from Blockchain challenge object. - CreatedAt common.Timestamp `gorm:"created_at" json:"created"` - UpdatedAt time.Time `gorm:"updated_at;type:timestamp without time zone;not null;default:current_timestamp" json:"-"` + RoundCreatedAt int64 `gorm:"round_created_at" json:"round_created_at"` + CreatedAt common.Timestamp `gorm:"created_at" json:"created"` + UpdatedAt time.Time `gorm:"updated_at;type:timestamp without time zone;not null;default:current_timestamp" json:"-"` } func (ChallengeEntity) TableName() string { @@ -140,7 +141,7 @@ func unMarshalField(stringObj datatypes.JSON, dest interface{}) error { func (cr *ChallengeEntity) Save(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) - return cr.SaveWith(db) + return cr.SaveWith(db.DB) } func (cr *ChallengeEntity) SaveWith(db *gorm.DB) error { diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index aa4423f8c..9f5ecac0e 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -48,7 +48,7 @@ type ChallengeResponse struct { func (cr *ChallengeEntity) CancelChallenge(ctx context.Context, errReason error) { cancellation := time.Now() db := datastore.GetStore().GetTransaction(ctx) - deleteChallenge(int64(cr.CreatedAt)) + deleteChallenge(cr.RoundCreatedAt) cr.Status = Cancelled cr.StatusMessage = errReason.Error() cr.UpdatedAt = cancellation.UTC() diff --git a/code/go/0chain.net/blobbercore/challenge/timing.go b/code/go/0chain.net/blobbercore/challenge/timing.go index 80282d08e..938e237f7 100644 --- a/code/go/0chain.net/blobbercore/challenge/timing.go +++ b/code/go/0chain.net/blobbercore/challenge/timing.go @@ -167,3 +167,10 @@ func GetChallengeTimings(from common.Timestamp, limit common.Pagination) ([]*Cha } return chs, nil } + +func GetChallengeTiming(challengeID string) (*ChallengeTiming, error) { + var ch *ChallengeTiming + + err := datastore.GetStore().GetDB().Model(&ChallengeTiming{}).Where("challenge_id = ?", challengeID).First(&ch).Error + return ch, err +} diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go index 4afc61c56..509500819 100644 --- a/code/go/0chain.net/blobbercore/challenge/worker.go +++ b/code/go/0chain.net/blobbercore/challenge/worker.go @@ -6,6 +6,7 @@ import ( "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/emirpasic/gods/maps/treemap" "go.uber.org/zap" @@ -81,6 +82,8 @@ func challengeProcessor(ctx context.Context) { return case it := <-toProcessChallenge: + + logging.Logger.Info("processing_challenge", zap.Any("challenge_id", it.ChallengeID)) if ok := it.createChallenge(); !ok { continue } @@ -142,7 +145,7 @@ func commitOnChainWorker(ctx context.Context) { }() err := challenge.VerifyChallengeTransaction(txn) if err == nil || err != ErrEntityNotFound { - deleteChallenge(int64(challenge.CreatedAt)) + deleteChallenge(int64(challenge.RoundCreatedAt)) } }(&chall) } @@ -175,12 +178,21 @@ func getBatch(batchSize int) (chall []ChallengeEntity) { func (it *ChallengeEntity) createChallenge() bool { challengeMapLock.Lock() - if _, ok := challengeMap.Get(int64(it.CreatedAt)); ok { - challengeMapLock.Unlock() + defer challengeMapLock.Unlock() + if _, ok := challengeMap.Get(it.RoundCreatedAt); ok { return false } - challengeMap.Put(int64(it.CreatedAt), it) - challengeMapLock.Unlock() + db := datastore.GetStore().GetDB() + var Found bool + err := db.Raw("SELECT EXISTS(SELECT 1 FROM challenge_timing WHERE challenge_id = ?) AS found", it.ChallengeID).Scan(&Found).Error + if err != nil { + logging.Logger.Error("createChallenge", zap.Error(err)) + return false + } else if Found { + logging.Logger.Info("createChallenge", zap.String("challenge_id", it.ChallengeID), zap.String("status", "already exists")) + return false + } + challengeMap.Put(it.RoundCreatedAt, it) return true } diff --git a/code/go/0chain.net/blobbercore/datastore/mocket.go b/code/go/0chain.net/blobbercore/datastore/mocket.go index 6575e8876..de975d32f 100644 --- a/code/go/0chain.net/blobbercore/datastore/mocket.go +++ b/code/go/0chain.net/blobbercore/datastore/mocket.go @@ -73,18 +73,22 @@ func (store *Mocket) Close() { func (store *Mocket) CreateTransaction(ctx context.Context) context.Context { db := store.db.Begin() - return context.WithValue(ctx, ContextKeyTransaction, db) + return context.WithValue(ctx, ContextKeyTransaction, EnhanceDB(db)) } -func (store *Mocket) GetTransaction(ctx context.Context) *gorm.DB { +func (store *Mocket) GetTransaction(ctx context.Context) *EnhancedDB { conn := ctx.Value(ContextKeyTransaction) if conn != nil { - return conn.(*gorm.DB) + return conn.(*EnhancedDB) } Logger.Error("No connection in the context.") return nil } +func (store *Mocket) WithTransaction(ctx context.Context, tx *gorm.DB) context.Context { + return context.WithValue(ctx, ContextKeyTransaction, EnhanceDB(tx)) +} + func (store *Mocket) GetDB() *gorm.DB { return store.db } diff --git a/code/go/0chain.net/blobbercore/datastore/postgres.go b/code/go/0chain.net/blobbercore/datastore/postgres.go index 41478e9f3..0d237aaac 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres.go @@ -85,18 +85,22 @@ func (store *postgresStore) Close() { func (store *postgresStore) CreateTransaction(ctx context.Context) context.Context { db := store.db.Begin() - return context.WithValue(ctx, ContextKeyTransaction, db) + return context.WithValue(ctx, ContextKeyTransaction, EnhanceDB(db)) } -func (store *postgresStore) GetTransaction(ctx context.Context) *gorm.DB { +func (store *postgresStore) GetTransaction(ctx context.Context) *EnhancedDB { conn := ctx.Value(ContextKeyTransaction) if conn != nil { - return conn.(*gorm.DB) + return conn.(*EnhancedDB) } logging.Logger.Error("No connection in the context.") return nil } +func (store *postgresStore) WithTransaction(ctx context.Context, tx *gorm.DB) context.Context { + return context.WithValue(ctx, ContextKeyTransaction, EnhanceDB(tx)) +} + func (store *postgresStore) GetDB() *gorm.DB { return store.db } diff --git a/code/go/0chain.net/blobbercore/datastore/sqlmock.go b/code/go/0chain.net/blobbercore/datastore/sqlmock.go index 606949625..f0fdc6a3f 100644 --- a/code/go/0chain.net/blobbercore/datastore/sqlmock.go +++ b/code/go/0chain.net/blobbercore/datastore/sqlmock.go @@ -71,15 +71,19 @@ func (store *Sqlmock) CreateTransaction(ctx context.Context) context.Context { return context.WithValue(ctx, ContextKeyTransaction, db) } -func (store *Sqlmock) GetTransaction(ctx context.Context) *gorm.DB { +func (store *Sqlmock) GetTransaction(ctx context.Context) *EnhancedDB { conn := ctx.Value(ContextKeyTransaction) if conn != nil { - return conn.(*gorm.DB) + return conn.(*EnhancedDB) } Logger.Error("No connection in the context.") return nil } +func (store *Sqlmock) WithTransaction(ctx context.Context, tx *gorm.DB) context.Context { + return context.WithValue(ctx, ContextKeyTransaction, EnhanceDB(tx)) +} + func (store *Sqlmock) GetDB() *gorm.DB { return store.db } diff --git a/code/go/0chain.net/blobbercore/datastore/store.go b/code/go/0chain.net/blobbercore/datastore/store.go index a950410e6..e24ca8804 100644 --- a/code/go/0chain.net/blobbercore/datastore/store.go +++ b/code/go/0chain.net/blobbercore/datastore/store.go @@ -13,6 +13,16 @@ const ( ContextKeyStore ) +type EnhancedDB struct { + SessionCache map[string]interface{} + *gorm.DB +} + +func EnhanceDB(db *gorm.DB) *EnhancedDB { + cache := make(map[string]interface{}) + return &EnhancedDB{DB: db, SessionCache: cache} +} + type Store interface { // GetDB get raw gorm db @@ -20,8 +30,8 @@ type Store interface { // CreateTransaction create transaction, and save it in context CreateTransaction(ctx context.Context) context.Context // GetTransaction get transaction from context - GetTransaction(ctx context.Context) *gorm.DB - + GetTransaction(ctx context.Context) *EnhancedDB + WithTransaction(ctx context.Context, tx *gorm.DB) context.Context // Get db connection with user that creates roles and databases. Its dialactor does not contain database name GetPgDB() (*gorm.DB, error) Open() error diff --git a/code/go/0chain.net/blobbercore/handler/authticket.go b/code/go/0chain.net/blobbercore/handler/authticket.go index a4d4d6a08..c96abc0ba 100644 --- a/code/go/0chain.net/blobbercore/handler/authticket.go +++ b/code/go/0chain.net/blobbercore/handler/authticket.go @@ -10,11 +10,10 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "gorm.io/gorm" ) // verifyAuthTicket verifies authTicket and returns authToken and error if any. For any error authToken is nil -func verifyAuthTicket(ctx context.Context, db *gorm.DB, authTokenString string, allocationObj *allocation.Allocation, refRequested *reference.Ref, clientID string, verifyShare bool) (*readmarker.AuthTicket, error) { +func verifyAuthTicket(ctx context.Context, authTokenString string, allocationObj *allocation.Allocation, refRequested *reference.Ref, clientID string, verifyShare bool) (*readmarker.AuthTicket, error) { if authTokenString == "" { return nil, common.NewError("invalid_parameters", "Auth ticket is required") } @@ -29,7 +28,7 @@ func verifyAuthTicket(ctx context.Context, db *gorm.DB, authTokenString string, } if refRequested.LookupHash != authToken.FilePathHash { - authTokenRef, err := reference.GetLimitedRefFieldsByLookupHashWith(ctx, db, authToken.AllocationID, authToken.FilePathHash, []string{"id", "path"}) + authTokenRef, err := reference.GetLimitedRefFieldsByLookupHashWith(ctx, authToken.AllocationID, authToken.FilePathHash, []string{"id", "path"}) if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/challenge_timings.go b/code/go/0chain.net/blobbercore/handler/challenge_timings.go index af02bf711..5ea951a5c 100644 --- a/code/go/0chain.net/blobbercore/handler/challenge_timings.go +++ b/code/go/0chain.net/blobbercore/handler/challenge_timings.go @@ -29,3 +29,11 @@ func GetChallengeTimings(ctx context.Context, r *http.Request) (interface{}, err return challenge.GetChallengeTimings(from, limit) } + +func GetChallengeTiming(ctx context.Context, r *http.Request) (interface{}, error) { + var ( + challengeID = r.URL.Query().Get("challenge_id") + ) + + return challenge.GetChallengeTiming(challengeID) +} diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index cb791b655..7e083f0cd 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -229,6 +229,7 @@ func setupHandlers(r *mux.Router) { r.HandleFunc("/getstats", RateLimitByCommmitRL(common.ToJSONResponse(stats.GetStatsHandler))) // r.HandleFunc("/challengetimings", common.AuthenticateAdmin(common.ToJSONResponse(GetChallengeTimings))) r.HandleFunc("/challengetimings", RateLimitByCommmitRL(common.ToJSONResponse(GetChallengeTimings))) + r.HandleFunc("/challenge-timings-by-challengeId", RateLimitByCommmitRL(common.ToJSONResponse(GetChallengeTiming))) //marketplace related r.HandleFunc("/v1/marketplace/shareinfo/{allocation}", diff --git a/code/go/0chain.net/blobbercore/handler/handler_playlist.go b/code/go/0chain.net/blobbercore/handler/handler_playlist.go index 0fd0505c1..4cb4637b3 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_playlist.go +++ b/code/go/0chain.net/blobbercore/handler/handler_playlist.go @@ -25,7 +25,7 @@ func LoadPlaylist(ctx *Context) (interface{}, error) { return nil, errors.New("lookup_hash_missed: auth_token and lookup_hash are required") } - fileRef, err := reference.GetLimitedRefFieldsByLookupHashWith(ctx, ctx.Store.GetDB(), ctx.AllocationId, lookupHash, []string{"id", "path", "lookup_hash", "type", "name"}) + fileRef, err := reference.GetLimitedRefFieldsByLookupHashWith(ctx, ctx.AllocationId, lookupHash, []string{"id", "path", "lookup_hash", "type", "name"}) if err != nil { return nil, common.NewError("invalid_lookup_hash", err.Error()) } @@ -35,7 +35,7 @@ func LoadPlaylist(ctx *Context) (interface{}, error) { return nil, common.NewError("invalid_auth_ticket", err.Error()) } - authToken, err := verifyAuthTicket(ctx, ctx.Store.GetDB(), string(at), ctx.Allocation, fileRef, ctx.ClientID, true) + authToken, err := verifyAuthTicket(ctx, string(at), ctx.Allocation, fileRef, ctx.ClientID, true) if err != nil { return nil, err } @@ -67,7 +67,7 @@ func LoadPlaylistFile(ctx *Context) (interface{}, error) { //load playlist with auth ticket if len(authTokenString) > 0 { - fileRef, err := reference.GetLimitedRefFieldsByLookupHashWith(ctx, ctx.Store.GetDB(), ctx.AllocationId, lookupHash, []string{"id", "path", "lookup_hash", "type", "name"}) + fileRef, err := reference.GetLimitedRefFieldsByLookupHashWith(ctx, ctx.AllocationId, lookupHash, []string{"id", "path", "lookup_hash", "type", "name"}) if err != nil { return nil, common.NewError("invalid_lookup_hash", err.Error()) } @@ -75,7 +75,7 @@ func LoadPlaylistFile(ctx *Context) (interface{}, error) { if err != nil { return nil, common.NewError("invalid_auth_ticket", err.Error()) } - authToken, err := verifyAuthTicket(ctx, ctx.Store.GetDB(), string(at), ctx.Allocation, fileRef, ctx.ClientID, true) + authToken, err := verifyAuthTicket(ctx, string(at), ctx.Allocation, fileRef, ctx.ClientID, true) if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/handler_writemarker.go b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go index f8dccbabc..5704862a8 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_writemarker.go +++ b/code/go/0chain.net/blobbercore/handler/handler_writemarker.go @@ -3,6 +3,8 @@ package handler import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/common" + . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" ) var WriteMarkerMutext = &writemarker.Mutex{ @@ -14,6 +16,7 @@ func LockWriteMarker(ctx *Context) (interface{}, error) { connectionID, _ := ctx.FormValue("connection_id") result, err := WriteMarkerMutext.Lock(ctx, ctx.AllocationId, connectionID) + Logger.Info("Lock write marker result", zap.Any("result", result), zap.Error(err)) if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index e2f8d4c08..4e58e8526 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -27,10 +27,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/node" - "gorm.io/gorm" - "gorm.io/gorm/clause" - "go.uber.org/zap" + "gorm.io/gorm" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" ) @@ -52,7 +50,6 @@ func readPreRedeem( // check out read pool tokens if read_price > 0 var ( - db = datastore.GetStore().GetTransaction(ctx) blobberID = node.Self.ID ) @@ -60,7 +57,7 @@ func readPreRedeem( return // skip if read price is zero } - readPoolBalance, err := allocation.GetReadPoolsBalance(db, payerID) + readPoolBalance, err := allocation.GetReadPoolsBalance(ctx, payerID) if err != nil { return common.NewError("read_pre_redeem", "database error while reading read pools balance") } @@ -73,7 +70,7 @@ func readPreRedeem( } rp.ClientID = payerID - err = allocation.UpdateReadPool(db, rp) + err = allocation.UpdateReadPool(ctx, rp) if err != nil { return common.NewErrorf("read_pre_redeem", "can't save requested read pools: %v", err) } @@ -108,7 +105,6 @@ func checkPendingMarkers(ctx context.Context, allocationID string) error { func writePreRedeem(ctx context.Context, alloc *allocation.Allocation, writeMarker *writemarker.WriteMarker, payerID string) (err error) { // check out read pool tokens if read_price > 0 var ( - db = datastore.GetStore().GetTransaction(ctx) blobberID = node.Self.ID requiredBalance = alloc.GetRequiredWriteBalance(blobberID, writeMarker.Size, writeMarker.Timestamp) wp *allocation.WritePool @@ -118,13 +114,13 @@ func writePreRedeem(ctx context.Context, alloc *allocation.Allocation, writeMark return } - writePoolBalance, err := allocation.GetWritePoolsBalance(db, alloc.ID) + writePoolBalance, err := allocation.GetWritePoolsBalance(ctx, alloc.ID) if err != nil { Logger.Error("write_pre_redeem:get_write_pools_balance", zap.Error(err), zap.String("allocation_id", alloc.ID)) return common.NewError("write_pre_redeem", "database error while getting write pool balance") } - pendingWriteSize, err := allocation.GetPendingWrite(db, payerID, alloc.ID) + pendingWriteSize, err := allocation.GetPendingWrite(ctx, payerID, alloc.ID) if err != nil { escapedPayerID := sanitizeString(payerID) Logger.Error("write_pre_redeem:get_pending_write", zap.Error(err), zap.String("allocation_id", alloc.ID), zap.String("payer_id", escapedPayerID)) @@ -139,7 +135,7 @@ func writePreRedeem(ctx context.Context, alloc *allocation.Allocation, writeMark return common.NewErrorf("write_pre_redeem", "can't request write pools from sharders: %v", err) } - err = allocation.SetWritePool(db, alloc.ID, wp) + err = allocation.SetWritePool(ctx, alloc.ID, wp) if err != nil { return common.NewErrorf("write_pre_redeem", "can't save requested write pools: %v", err) } @@ -154,7 +150,7 @@ func writePreRedeem(ctx context.Context, alloc *allocation.Allocation, writeMark alloc.ID, writeMarker.BlobberID, writePoolBalance, requiredBalance) } - if err := allocation.AddToPending(db, payerID, alloc.ID, writeMarker.Size); err != nil { + if err := allocation.AddToPending(ctx, payerID, alloc.ID, writeMarker.Size); err != nil { Logger.Error(err.Error()) return common.NewErrorf("write_pre_redeem", "can't save pending writes in DB") @@ -654,17 +650,16 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b if err = db.Create(writemarkerEntity).Error; err != nil { return nil, common.NewError("write_marker_error", "Error persisting the write marker") } + allocationObj.AllocationRoot = allocationRoot + allocationObj.FileMetaRoot = fileMetaRoot + allocationObj.IsRedeemRequired = true + allocationObj.BlobberSizeUsed += connectionObj.Size + allocationObj.UsedSize += connectionObj.Size - allocationUpdates := make(map[string]interface{}) - allocationUpdates["blobber_size_used"] = gorm.Expr("blobber_size_used + ?", connectionObj.Size) - allocationUpdates["used_size"] = gorm.Expr("used_size + ?", connectionObj.Size) - allocationUpdates["allocation_root"] = allocationRoot - allocationUpdates["file_meta_root"] = fileMetaRoot - allocationUpdates["is_redeem_required"] = true - - if err = db.Model(allocationObj).Updates(allocationUpdates).Error; err != nil { + if err = allocation.Repo.Save(ctx, allocationObj); err != nil { return nil, common.NewError("allocation_write_error", "Error persisting the allocation object") } + err = connectionObj.CommitToFileStore(ctx) if err != nil { if !errors.Is(common.ErrFileWasDeleted, err) { @@ -1361,16 +1356,20 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob } elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM - - err = allocation.ApplyRollback(ctx, allocationID) + c := datastore.GetStore().CreateTransaction(context.TODO()) + defer c.Done() + txn := datastore.GetStore().GetTransaction(c) + err = allocation.ApplyRollback(c, allocationID) if err != nil { + txn.Rollback() return nil, common.NewError("allocation_rollback_error", "Error applying the rollback for allocation: "+err.Error()) } elapsedApplyRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM - elapsedWritePreRedeem //get allocation root and ref - rootRef, err := reference.GetLimitedRefFieldsByPath(ctx, allocationID, "/", []string{"hash", "file_meta_hash", "is_precommit"}) + rootRef, err := reference.GetLimitedRefFieldsByPath(c, allocationID, "/", []string{"hash", "file_meta_hash", "is_precommit"}) if err != nil && err != gorm.ErrRecordNotFound { + txn.Rollback() return nil, common.NewError("root_ref_read_error", "Error reading the root reference: "+err.Error()) } if err == gorm.ErrRecordNotFound { @@ -1389,6 +1388,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob result.Success = false result.ErrorMessage = "Allocation root in the write marker does not match the calculated allocation root." + " Expected hash: " + allocationRoot + txn.Rollback() return &result, common.NewError("allocation_root_mismatch", result.ErrorMessage) } @@ -1399,6 +1399,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob result.Success = false result.ErrorMessage = "File meta root in the write marker does not match the calculated file meta root." + " Expected hash: " + fileMetaRoot + "; Got: " + writeMarker.FileMetaRoot + txn.Rollback() return &result, common.NewError("file_meta_root_mismatch", result.ErrorMessage) } @@ -1406,42 +1407,36 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob writemarkerEntity.ClientPublicKey = clientKey Logger.Info("rollback_writemarker", zap.Any("writemarker", writemarkerEntity.WM)) - db := datastore.GetStore().GetDB() - alloc := &allocation.Allocation{} - - err = db.Transaction(func(tx *gorm.DB) error { - - err := tx.Model(&allocation.Allocation{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("is_redeem_required").Where("id=?", allocationID).First(alloc).Error - if err != nil { - return common.NewError("allocation_read_error", "Error reading the allocation object") - } - - allocationUpdates := make(map[string]interface{}) - allocationUpdates["blobber_size_used"] = gorm.Expr("blobber_size_used - ?", latestWriteMarkerEntity.WM.Size) - allocationUpdates["used_size"] = gorm.Expr("used_size - ?", latestWriteMarkerEntity.WM.Size) - allocationUpdates["is_redeem_required"] = true - allocationUpdates["allocation_root"] = allocationRoot - allocationUpdates["file_meta_root"] = fileMetaRoot + alloc, err := allocation.Repo.GetByIdAndLock(c, allocationID) + if err != nil { + txn.Rollback() + return &result, common.NewError("allocation_read_error", "Error reading the allocation object") + } - if alloc.IsRedeemRequired { - writemarkerEntity.Status = writemarker.Rollbacked - allocationUpdates["is_redeem_required"] = false - } - err = tx.Create(writemarkerEntity).Error - if err != nil { - return common.NewError("write_marker_error", "Error persisting the write marker "+err.Error()) - } + alloc.BlobberSizeUsed -= latestWriteMarkerEntity.WM.Size + alloc.UsedSize -= latestWriteMarkerEntity.WM.Size + alloc.AllocationRoot = allocationRoot + alloc.FileMetaRoot = fileMetaRoot - if err = tx.Model(allocationObj).Updates(allocationUpdates).Error; err != nil { - return common.NewError("allocation_write_error", "Error persisting the allocation object "+err.Error()) - } + if alloc.IsRedeemRequired { + writemarkerEntity.Status = writemarker.Rollbacked + alloc.IsRedeemRequired = false + } + err = txn.Create(writemarkerEntity).Error + if err != nil { + txn.Rollback() + return &result, common.NewError("write_marker_error", "Error persisting the write marker "+err.Error()) + } + if err = allocation.Repo.Save(c, alloc); err != nil { + txn.Rollback() + return &result, common.NewError("allocation_write_error", "Error persisting the allocation object "+err.Error()) + } - return nil - }) + err = txn.Commit().Error if err != nil { - return nil, err + return &result, common.NewError("allocation_commit_error", "Error committing the transaction "+err.Error()) } - if !alloc.IsRedeemRequired { + if alloc.IsRedeemRequired { err = writemarkerEntity.SendToChan(ctx) if err != nil { return nil, common.NewError("write_marker_error", "Error redeeming the write marker") diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go index 201ceda0c..ced24036d 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go @@ -290,8 +290,8 @@ func TestDownloadFile(t *testing.T) { ctx = context.WithValue(ctx, constants.ContextKeyAllocation, p.inData.allocationTx) ctx = context.WithValue(ctx, constants.ContextKeyClientKey, client.GetClientPublicKey()) - db := datastore.GetStore().GetDB().Begin() - ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db) + ctx = datastore.GetStore().CreateTransaction(ctx) + return ctx } diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 557294497..8e3cfad68 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -9,11 +9,11 @@ import ( "regexp" "strconv" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" - "github.com/0chain/gosdk/constants" "go.uber.org/zap" @@ -50,6 +50,8 @@ func (fsh *StorageHandler) verifyAllocation(ctx context.Context, allocationID, a "verifying allocation transaction error: %v", err) } + logging.Logger.Info("verifyAllocation", zap.Any("alloc", alloc), zap.Any("now", common.Now())) + if alloc.Expiration < common.Now() { return nil, common.NewError("verify_allocation", "use of expired allocation") @@ -70,10 +72,7 @@ func (fsh *StorageHandler) convertGormError(err error) error { // verifyAuthTicket verifies authTicket and returns authToken and error if any. For any error authToken is nil func (fsh *StorageHandler) verifyAuthTicket(ctx context.Context, authTokenString string, allocationObj *allocation.Allocation, refRequested *reference.Ref, clientID string, verifyShare bool) (*readmarker.AuthTicket, error) { - - db := datastore.GetStore().GetTransaction(ctx) - - return verifyAuthTicket(ctx, db, authTokenString, allocationObj, refRequested, clientID, verifyShare) + return verifyAuthTicket(ctx, authTokenString, allocationObj, refRequested, clientID, verifyShare) } func (fsh *StorageHandler) GetAllocationDetails(ctx context.Context, r *http.Request) (interface{}, error) { diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go index 127f90812..2634b148e 100644 --- a/code/go/0chain.net/blobbercore/handler/worker.go +++ b/code/go/0chain.net/blobbercore/handler/worker.go @@ -4,14 +4,12 @@ import ( "context" "time" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" - "github.com/0chain/blobber/code/go/0chain.net/core/lock" - "gorm.io/gorm" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" + "github.com/0chain/blobber/code/go/0chain.net/core/lock" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "go.uber.org/zap" @@ -27,15 +25,16 @@ func CleanupDiskFiles(ctx context.Context) error { db.Find(&allocations) for _, allocationObj := range allocations { - cleanupAllocationFiles(db, allocationObj) + cleanupAllocationFiles(ctx, allocationObj) } return nil } -func cleanupAllocationFiles(db *gorm.DB, allocationObj allocation.Allocation) { +func cleanupAllocationFiles(ctx context.Context, allocationObj allocation.Allocation) { mutex := lock.GetMutex(allocationObj.TableName(), allocationObj.ID) mutex.Lock() defer mutex.Unlock() + db := datastore.GetStore().GetTransaction(ctx) _ = filestore.GetFileStore().IterateObjects(allocationObj.ID, func(hash string, contentSize int64) { var refs []reference.Ref diff --git a/code/go/0chain.net/blobbercore/readmarker/readmarker.go b/code/go/0chain.net/blobbercore/readmarker/readmarker.go index e49ec747a..7d90819ef 100644 --- a/code/go/0chain.net/blobbercore/readmarker/readmarker.go +++ b/code/go/0chain.net/blobbercore/readmarker/readmarker.go @@ -184,7 +184,6 @@ func SaveLatestReadMarker(ctx context.Context, rm *ReadMarker, latestRedeemedRC // Sync read marker with 0chain to be sure its correct. func (rm *ReadMarkerEntity) Sync(ctx context.Context) error { - var db = datastore.GetStore().GetTransaction(ctx) // update local read pools cache from sharders rp, err := allocation.RequestReadPoolStat(rm.LatestRM.ClientID) if err != nil { @@ -192,7 +191,7 @@ func (rm *ReadMarkerEntity) Sync(ctx context.Context) error { } // save the fresh read pools information - err = allocation.UpsertReadPool(db, rp) + err = allocation.UpsertReadPool(ctx, rp) if err != nil { return common.NewErrorf("rme_sync", "can't update read pools from sharders: %v", err) } @@ -222,7 +221,7 @@ func (rme *ReadMarkerEntity) UpdateStatus(ctx context.Context, txOutput, redeemT return common.NewErrorf("rme_update_status", "can't get read pools from sharders: %v", err) } - if err := allocation.UpdateReadPool(db, rp); err != nil { + if err := allocation.UpdateReadPool(ctx, rp); err != nil { return common.NewErrorf("rme_update_status", "can't update local read pools cache: %v", err) } diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index aab633054..7cce69d95 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -193,8 +193,8 @@ func GetReference(ctx context.Context, allocationID, path string) (*Ref, error) // GetLimitedRefFieldsByPath get FileRef selected fields with allocationID and path from postgres func GetLimitedRefFieldsByPath(ctx context.Context, allocationID, path string, selectedFields []string) (*Ref, error) { ref := &Ref{} - db := datastore.GetStore().GetTransaction(ctx) - db = db.Select(selectedFields) + t := datastore.GetStore().GetTransaction(ctx) + db := t.Select(selectedFields) err := db.Where(&Ref{AllocationID: allocationID, Path: path}).First(ref).Error if err != nil { return nil, err @@ -203,8 +203,9 @@ func GetLimitedRefFieldsByPath(ctx context.Context, allocationID, path string, s } // GetLimitedRefFieldsByLookupHash get FileRef selected fields with allocationID and lookupHash from postgres -func GetLimitedRefFieldsByLookupHashWith(ctx context.Context, db *gorm.DB, allocationID, lookupHash string, selectedFields []string) (*Ref, error) { +func GetLimitedRefFieldsByLookupHashWith(ctx context.Context, allocationID, lookupHash string, selectedFields []string) (*Ref, error) { ref := &Ref{} + db := datastore.GetStore().GetTransaction(ctx) err := db. Select(selectedFields). @@ -220,8 +221,8 @@ func GetLimitedRefFieldsByLookupHashWith(ctx context.Context, db *gorm.DB, alloc // GetLimitedRefFieldsByLookupHash get FileRef selected fields with allocationID and lookupHash from postgres func GetLimitedRefFieldsByLookupHash(ctx context.Context, allocationID, lookupHash string, selectedFields []string) (*Ref, error) { ref := &Ref{} - db := datastore.GetStore().GetTransaction(ctx) - db = db.Select(selectedFields) + t := datastore.GetStore().GetTransaction(ctx) + db := t.Select(selectedFields) err := db.Where(&Ref{AllocationID: allocationID, LookupHash: lookupHash}).First(ref).Error if err != nil { return nil, err @@ -286,8 +287,8 @@ func GetRefsTypeFromPaths(ctx context.Context, allocationID string, paths []stri return } - db := datastore.GetStore().GetTransaction(ctx) - db = db.Select("path", "type") + t := datastore.GetStore().GetTransaction(ctx) + db := t.Select("path", "type") for _, p := range paths { db = db.Or(Ref{AllocationID: allocationID, Path: p}) } @@ -314,8 +315,8 @@ func GetSubDirsFromPath(p string) []string { func GetRefWithChildren(ctx context.Context, allocationID, path string) (*Ref, error) { var refs []Ref - db := datastore.GetStore().GetTransaction(ctx) - db = db.Where(Ref{ParentPath: path, AllocationID: allocationID}).Or(Ref{Type: DIRECTORY, Path: path, AllocationID: allocationID}) + t := datastore.GetStore().GetTransaction(ctx) + db := t.Where(Ref{ParentPath: path, AllocationID: allocationID}).Or(Ref{Type: DIRECTORY, Path: path, AllocationID: allocationID}) err := db.Order("path").Find(&refs).Error if err != nil { return nil, err @@ -339,8 +340,8 @@ func GetRefWithChildren(ctx context.Context, allocationID, path string) (*Ref, e func GetRefWithSortedChildren(ctx context.Context, allocationID, path string) (*Ref, error) { var refs []*Ref - db := datastore.GetStore().GetTransaction(ctx) - db = db.Where( + t := datastore.GetStore().GetTransaction(ctx) + db := t.Where( Ref{ParentPath: path, AllocationID: allocationID}). Or(Ref{Type: DIRECTORY, Path: path, AllocationID: allocationID}) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index a9e9e60be..1f5176a1b 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -25,8 +25,8 @@ func GetReferencePath(ctx context.Context, allocationID, path string) (*Ref, err // GetReferenceForHashCalculationFromPaths validate and build full dir tree from db, and CalculateHash and return root Ref without saving in DB func GetReferenceForHashCalculationFromPaths(ctx context.Context, allocationID string, paths []string) (*Ref, error) { var refs []Ref - db := datastore.GetStore().GetTransaction(ctx) - db = db.Model(&Ref{}).Select("id", "allocation_id", "type", "name", "path", "parent_path", "size", "hash", "file_meta_hash", + t := datastore.GetStore().GetTransaction(ctx) + db := t.Model(&Ref{}).Select("id", "allocation_id", "type", "name", "path", "parent_path", "size", "hash", "file_meta_hash", "path_hash", "validation_root", "fixed_merkle_root", "actual_file_size", "actual_file_hash", "chunk_size", "lookup_hash", "thumbnail_hash", "allocation_root", "level", "created_at", "updated_at", "file_id") @@ -122,7 +122,9 @@ func (rootRef *Ref) GetSrcPath(path string) (*Ref, error) { // GetReferencePathFromPaths validate and build full dir tree from db, and CalculateHash and return root Ref func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths, objTreePath []string) (*Ref, error) { var refs []Ref - db := datastore.GetStore().GetTransaction(ctx) + t := datastore.GetStore().GetTransaction(ctx) + db := t.DB + pathsAdded := make(map[string]bool) var shouldOr bool for _, path := range paths { @@ -199,8 +201,8 @@ func GetReferencePathFromPaths(ctx context.Context, allocationID string, paths, func GetObjectTree(ctx context.Context, allocationID, path string) (*Ref, error) { path = filepath.Clean(path) var refs []Ref - db := datastore.GetStore().GetTransaction(ctx) - db = db.Where(Ref{Path: path, AllocationID: allocationID}) + t := datastore.GetStore().GetTransaction(ctx) + db := t.Where(Ref{Path: path, AllocationID: allocationID}) if path != "/" { db = db.Or("path LIKE ? AND allocation_id = ?", path+"/%", allocationID) } else { diff --git a/code/go/0chain.net/blobbercore/writemarker/entity.go b/code/go/0chain.net/blobbercore/writemarker/entity.go index cbe12e1d0..e99e5d01c 100644 --- a/code/go/0chain.net/blobbercore/writemarker/entity.go +++ b/code/go/0chain.net/blobbercore/writemarker/entity.go @@ -101,7 +101,7 @@ func (wm *WriteMarkerEntity) UpdateStatus(ctx context.Context, status WriteMarke } // work on pre-redeemed tokens and write-pools balances tracking - if err := allocation.AddToPending(db, wm.WM.ClientID, wm.WM.AllocationID, -wm.WM.Size); err != nil { + if err := allocation.AddToPending(ctx, wm.WM.ClientID, wm.WM.AllocationID, -wm.WM.Size); err != nil { return fmt.Errorf("can't save allocation pending value: %v", err) } return diff --git a/code/go/0chain.net/blobbercore/writemarker/mutex.go b/code/go/0chain.net/blobbercore/writemarker/mutex.go index 474fd1ae4..f3d56c296 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutex.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutex.go @@ -7,8 +7,10 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/errors" "github.com/0chain/gosdk/constants" + "go.uber.org/zap" "gorm.io/gorm" ) @@ -37,6 +39,7 @@ type Mutex struct { // If lock exists and is of same connection ID then lock's createdAt is updated // If lock exists and is of other connection ID then `pending` response is sent. func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string) (*LockResult, error) { + logging.Logger.Info("Locking write marker", zap.String("allocation_id", allocationID), zap.String("connection_id", connectionID)) if allocationID == "" { return nil, errors.Throw(constants.ErrInvalidParameter, "allocationID") } @@ -55,6 +58,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string) (*L err := db.Table(TableNameWriteLock).Where("allocation_id=?", allocationID).First(&lock).Error if err != nil { // new lock + logging.Logger.Info("Creating new lock") if errors.Is(err, gorm.ErrRecordNotFound) { lock = WriteLock{ AllocationID: allocationID, @@ -72,7 +76,7 @@ func (m *Mutex) Lock(ctx context.Context, allocationID, connectionID string) (*L CreatedAt: lock.CreatedAt.Unix(), }, nil } - + logging.Logger.Error("Could not create lock") //native postgres error return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore) } diff --git a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go index 03909b368..5055576ad 100644 --- a/code/go/0chain.net/blobbercore/writemarker/mutext_test.go +++ b/code/go/0chain.net/blobbercore/writemarker/mutext_test.go @@ -8,10 +8,16 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" gomocket "github.com/selvatico/go-mocket" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) +func init() { + logging.Logger = zap.NewNop() +} + func TestMutext_LockShouldWork(t *testing.T) { datastore.UseMocket(false) diff --git a/code/go/0chain.net/blobbercore/writemarker/worker.go b/code/go/0chain.net/blobbercore/writemarker/worker.go index da82e3fa2..ee73ac551 100644 --- a/code/go/0chain.net/blobbercore/writemarker/worker.go +++ b/code/go/0chain.net/blobbercore/writemarker/worker.go @@ -8,11 +8,9 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" "golang.org/x/sync/semaphore" "gorm.io/gorm" - "gorm.io/gorm/clause" - - "go.uber.org/zap" ) var ( @@ -22,14 +20,14 @@ var ( ) func SetupWorkers(ctx context.Context) { - db := datastore.GetStore().GetDB() - type Res struct { - ID string - } - var res []Res + var res []allocation.Res - err := db.Model(&allocation.Allocation{}).Select("id").Find(&res).Error + err := db.Transaction(func(tx *gorm.DB) error { + c := datastore.GetStore().WithTransaction(ctx, tx) + res = allocation.Repo.GetAllocationIds(c) + return nil + }) if err != nil && err != gorm.ErrRecordNotFound { logging.Logger.Error("error_getting_allocations_worker", zap.Any("error", err)) @@ -72,8 +70,7 @@ func redeemWriteMarker(wm *WriteMarkerEntity) error { } } }() - alloc := &allocation.Allocation{} - err := db.Model(&allocation.Allocation{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("allocation_root").Where("id=?", allocationID).First(alloc).Error + alloc, err := allocation.Repo.GetByIdAndLock(ctx, allocationID) if err != nil { logging.Logger.Error("Error redeeming the write marker.", zap.Any("allocation", allocationID), zap.Any("wm", wm.WM.AllocationID), zap.Any("error", err)) go tryAgain(wm) @@ -110,8 +107,7 @@ func redeemWriteMarker(wm *WriteMarkerEntity) error { mut.Release(1) } - err = db.Exec("UPDATE allocations SET latest_redeemed_write_marker=?,is_redeem_required=? WHERE id=?", - wm.WM.AllocationRoot, false, allocationID).Error + err = allocation.Repo.UpdateAllocationRedeem(ctx, wm.WM.AllocationRoot, allocationID) if err != nil { logging.Logger.Error("Error redeeming the write marker. Allocation latest wm redeemed update failed", diff --git a/docker.local/.dockerignore b/docker.local/.dockerignore index 628d1ada6..55bfc88ca 100644 --- a/docker.local/.dockerignore +++ b/docker.local/.dockerignore @@ -1,8 +1,13 @@ passphrase.txt logs/ +docs/ .git *.md .cache awsnet docker.local/blobber* +docker.aws/* +config/ +docker-clean/ +keys_config **/pkg \ No newline at end of file diff --git a/docker.local/b0docker-compose-debug.yml b/docker.local/b0docker-compose-debug.yml index 4b39e5194..99d577073 100644 --- a/docker.local/b0docker-compose-debug.yml +++ b/docker.local/b0docker-compose-debug.yml @@ -14,26 +14,10 @@ services: - ./sql_init:/docker-entrypoint-initdb.d networks: default: - postgres-post: - image: postgres:14 - environment: - POSTGRES_PORT: 5432 - POSTGRES_HOST: postgres - POSTGRES_USER: postgres - volumes: - - ../bin:/blobber/bin - - ../sql:/blobber/sql - command: bash /blobber/bin/postgres-entrypoint.sh - links: - - postgres:postgres validator: image: validator environment: - DOCKER= true - depends_on: - - postgres-post - links: - - postgres-post:postgres-post volumes: - ../config:/validator/config - ./blobber${BLOBBER}/data:/validator/data diff --git a/docker.local/b0docker-compose-github.yml b/docker.local/b0docker-compose-github.yml index a8e82ce93..838981ebf 100644 --- a/docker.local/b0docker-compose-github.yml +++ b/docker.local/b0docker-compose-github.yml @@ -14,18 +14,6 @@ services: - ./sql_init:/docker-entrypoint-initdb.d networks: default: - postgres-post: - image: postgres:14 - environment: - POSTGRES_PORT: 5432 - POSTGRES_HOST: postgres - POSTGRES_USER: postgres - volumes: - - ../bin:/blobber/bin - - ../sql:/blobber/sql - command: bash /blobber/bin/postgres-entrypoint.sh - links: - - postgres:postgres networks: default: diff --git a/docker.local/b0docker-compose.yml b/docker.local/b0docker-compose.yml index fee51770a..05bc5aa32 100644 --- a/docker.local/b0docker-compose.yml +++ b/docker.local/b0docker-compose.yml @@ -14,27 +14,10 @@ services: - ./sql_init:/docker-entrypoint-initdb.d networks: default: - postgres-post: - image: postgres:14 - environment: - POSTGRES_PORT: 5432 - POSTGRES_HOST: postgres - POSTGRES_USER: postgres - POSTGRES_PASSWORD: secret - volumes: - - ../bin:/blobber/bin - - ../sql:/blobber/sql - command: bash /blobber/bin/postgres-entrypoint.sh - links: - - postgres:postgres validator: image: validator environment: - DOCKER= true - depends_on: - - postgres-post - links: - - postgres-post:postgres-post volumes: - ${CONFIG_PATH:-../config}:/validator/config #value after :- is default value - ./blobber${BLOBBER}/data:/validator/data diff --git a/docker.local/docker-compose.yml b/docker.local/docker-compose.yml index 6ff4cfb98..56d4cb4d4 100644 --- a/docker.local/docker-compose.yml +++ b/docker.local/docker-compose.yml @@ -11,28 +11,10 @@ services: - "543${BLOBBER}:5432" labels: zchain: "postgres" - postgres-post: - image: postgres:14 - environment: - POSTGRES_PORT: 5432 - POSTGRES_HOST: postgres - POSTGRES_USER: postgres - volumes: - - ../bin:/blobber/bin - - ../sql:/blobber/sql - labels: - zchain: "postgres-post" - command: bash /blobber/bin/postgres-entrypoint.sh - links: - - postgres:postgres validator: image: validator environment: - DOCKER= true - depends_on: - - postgres-post - links: - - postgres-post:postgres-post volumes: - ../config:/validator/config - ./blobber${BLOBBER}/data:/validator/data diff --git a/docker.local/https.docker-compose.yml b/docker.local/https.docker-compose.yml index a2cf562bc..bd146aee6 100644 --- a/docker.local/https.docker-compose.yml +++ b/docker.local/https.docker-compose.yml @@ -4,34 +4,17 @@ services: image: postgres:14 volumes: - ./blobber${BLOBBER}/data/postgresql:/var/lib/postgresql/data + - ./sql_init:/docker-entrypoint-initdb.d networks: default: ports: - "543${BLOBBER}:5432" labels: zchain: "postgres" - postgres-post: - image: postgres:14 - environment: - POSTGRES_PORT: 5432 - POSTGRES_HOST: postgres - POSTGRES_USER: postgres - volumes: - - ../bin:/blobber/bin - - ../sql:/blobber/sql - labels: - zchain: "postgres-post" - command: bash /blobber/bin/postgres-entrypoint.sh - links: - - postgres:postgres validator: image: validator environment: - DOCKER= true - depends_on: - - postgres-post - links: - - postgres-post:postgres-post volumes: - ../config:/validator/config - ./blobber${BLOBBER}/data:/validator/data diff --git a/go.mod b/go.mod index d2b1f2958..df39f01df 100644 --- a/go.mod +++ b/go.mod @@ -21,12 +21,12 @@ require ( github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.10.0 + golang.org/x/crypto v0.11.0 golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.9.0 + golang.org/x/sys v0.10.0 golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect - google.golang.org/grpc v1.56.1 + google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect + google.golang.org/grpc v1.56.2 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -44,7 +44,7 @@ require ( require google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect -require github.com/pressly/goose/v3 v3.13.1 +require github.com/pressly/goose/v3 v3.13.4 require ( github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 // indirect @@ -85,11 +85,11 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.4.0 // indirect + github.com/jackc/pgx/v5 v5.4.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.16.6 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/klauspost/reedsolomon v1.11.7 // indirect github.com/machinebox/graphql v0.2.2 // indirect @@ -120,7 +120,7 @@ require ( go.mongodb.org/mongo-driver v1.11.3 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/text v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect diff --git a/go.sum b/go.sum index 10a27165f..745928389 100644 --- a/go.sum +++ b/go.sum @@ -435,8 +435,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.4.0 h1:BSr+GCm4N6QcgIwv0DyTFHK9ugfEFF9DzSbbzxOiXU0= -github.com/jackc/pgx/v5 v5.4.0/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= +github.com/jackc/pgx/v5 v5.4.1 h1:oKfB/FhuVtit1bBM3zNRRsZ925ZkMN3HXL+LgLUM9lE= +github.com/jackc/pgx/v5 v5.4.1/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -473,8 +473,8 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= -github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/reedsolomon v1.11.7 h1:9uaHU0slncktTEEg4+7Vl7q7XUNMBUOK4R9gnKhMjAU= @@ -524,7 +524,7 @@ github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/microsoft/go-mssqldb v1.1.0 h1:jsV+tpvcPTbNNKW0o3kiCD69kOHICsfjZ2VcVu2lKYc= +github.com/microsoft/go-mssqldb v1.3.0 h1:JcPVl+acL8Z/cQcJc9zP0OkjQ+l20bco/cCDpMbmGJk= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -604,8 +604,8 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pressly/goose/v3 v3.13.1 h1:4kYsfsboJvQDg1MEcMnXs8ubbz4AeHHXMW19XEAy3vI= -github.com/pressly/goose/v3 v3.13.1/go.mod h1:1WCIzQuOGhpx+D7bal+tGadbwVPmdfy9fltqYLMA8lM= +github.com/pressly/goose/v3 v3.13.4 h1:9xRcg/hEU9HqeRNeKh69VLtPWCKAYTX6l2VsXWOX86A= +github.com/pressly/goose/v3 v3.13.4/go.mod h1:Fo8rYaf9tYfQiDpo+ymrnZi8vvLkvguRl16nu7QnUT4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -631,7 +631,7 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= @@ -798,8 +798,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -835,7 +835,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -974,8 +974,8 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -986,8 +986,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1156,8 +1156,8 @@ google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= -google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= +google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1234,7 +1234,7 @@ lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q= modernc.org/ccgo/v3 v3.16.14 h1:af6KNtFgsVmnDYrWk3PQCS9XT6BXe7o3ZFJKkIKvXNQ= modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= -modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/memory v1.6.0 h1:i6mzavxrE9a30whzMfwf7XWVODx2r5OYXvU46cirX7o= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= diff --git a/goose/migrations/001_blobber_meta.sql b/goose/migrations/001_blobber_meta.sql index 7b5d72ec1..9eb80ae34 100644 --- a/goose/migrations/001_blobber_meta.sql +++ b/goose/migrations/001_blobber_meta.sql @@ -140,6 +140,7 @@ CREATE TABLE public.challenges ( object_path jsonb, sequence bigint, "timestamp" bigint DEFAULT 0 NOT NULL, + round_created_at bigint, created_at bigint, updated_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL );