Merge branch 'sprint-july-4' into add/case-2-and-case-4

0chain · Aug 4, 2023 · 73ded20 · 73ded20
2 parents 66319cf + 831c416
commit 73ded20
Show file tree

Hide file tree

Showing 50 changed files with 568 additions and 384 deletions.
diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml
@@ -16,6 +16,8 @@ env:
   BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }}
   VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }}
   DOCKER_CLI_EXPERIMENTAL: enabled
+  BLOBBER_BUILDBASE: blobber_base
+  BLOBBER_BUILD_BASE_REGISTRY: ${{ secrets.BLOBBER_BUILD_BASE_REGISTRY }}
 
 jobs:
   blobber:
@@ -39,7 +41,7 @@ jobs:
           go-version: ^1.20 # The Go version to download (if necessary) and use.
 
       - name: Clone blobber
-        uses: actions/checkout@v1
+        uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
         run: |
@@ -60,6 +62,26 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
+      - name: Get changed files using defaults
+        id: changed-files
+        uses: tj-actions/[email protected]
+
+      - name: Pull Build Base
+        run: |
+          docker pull $BLOBBER_BUILD_BASE_REGISTRY:staging
+          docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE
+
+      - name: Build Base image
+        if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile')
+        run: |
+          SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
+
+          ./docker.local/bin/build.base.sh &&
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+
       - name: Build blobber
         run: |
           SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
@@ -68,7 +90,7 @@ jobs:
           export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
           export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:${TAG} -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}"
           docker buildx create --driver-opt network=host --use --buildkitd-flags '--allow-insecure-entitlement security.insecure' --use blobber_buildx
-          ./docker.local/bin/build.base.sh && ./docker.local/bin/build.blobber.sh
+          ./docker.local/bin/build.blobber.sh
 
   validator:
     runs-on: [self-hosted, arc-runner]
@@ -112,14 +134,34 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
+      - name: Get changed files using defaults
+        id: changed-files
+        uses: tj-actions/[email protected]
+
+      - name: Pull Build Base
+        run: |
+          docker pull $BLOBBER_BUILD_BASE_REGISTRY:staging
+          docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE
+
+      - name: Build Base image
+        if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile')
+        run: |
+          SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
+
+          ./docker.local/bin/build.base.sh 
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+
       - name: Build validator
         run: |
           SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
           export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base"
           export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
           export DOCKER_IMAGE_VALIDATOR="-t ${VALIDATOR_REGISTRY}:${TAG} -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}"
           docker buildx create --driver-opt network=host --use --buildkitd-flags '--allow-insecure-entitlement security.insecure' --use blobber_buildx
-          ./docker.local/bin/build.base.sh && ./docker.local/bin/build.validator.sh
+          ./docker.local/bin/build.validator.sh
 
 
   system-tests:

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -6,10 +6,7 @@ concurrency:
 
 on:
   push:
-    branches:
-      - master
-      - staging
-    tags:
+    branches: [ master, staging, sprint* ]
   pull_request:
 
 jobs:

diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go
@@ -375,9 +375,7 @@ func TestBlobberCore_CopyFile(t *testing.T) {
 
 			config.Configuration.MaxAllocationDirFiles = tc.maxDirFilesPerAlloc
 
-			ctx := context.TODO()
-			db := datastore.GetStore().GetDB().Begin()
-			ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db)
+			ctx := datastore.GetStore().CreateTransaction(context.TODO())
 
 			change := &CopyFileChange{
 				AllocationID: tc.allocationID,

diff --git a/code/go/0chain.net/blobbercore/allocation/dao.go b/code/go/0chain.net/blobbercore/allocation/dao.go
@@ -2,6 +2,7 @@ package allocation
 
 import (
 	"context"
+
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
 	"github.com/0chain/blobber/code/go/0chain.net/core/common"
 	"github.com/0chain/errors"
@@ -12,33 +13,25 @@ import (
 // GetOrCreate, get allocation if it exists in db. if not, try to sync it from blockchain, and insert it in db.
 func GetOrCreate(ctx context.Context, store datastore.Store, allocationId string) (*Allocation, error) {
 
-	db := store.GetDB()
+	db := store.CreateTransaction(ctx)
 
 	if len(allocationId) == 0 {
 		return nil, errors.Throw(constants.ErrInvalidParameter, "tx")
 	}
 
-	alloc := &Allocation{}
-	result := db.Table(TableNameAllocation).Where(SQLWhereGetById, allocationId).First(alloc)
-
-	if result.Error == nil {
+	alloc, err := Repo.GetById(db, allocationId)
+	if err == nil {
 		return alloc, nil
 	}
+	if !errors.Is(err, gorm.ErrRecordNotFound) {
 
-	if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
-
-		return nil, errors.ThrowLog(result.Error.Error(), common.ErrBadDataStore)
+		return nil, errors.ThrowLog(err.Error(), common.ErrBadDataStore)
 	}
 
 	return SyncAllocation(allocationId)
 
 }
 
-const (
-	SQLWhereGetByTx = "allocations.tx = ?"
-	SQLWhereGetById = "allocations.id = ?"
-)
-
 // DryRun  Creates a prepared statement when executing any SQL and caches them to speed up future calls
 // https://gorm.io/docs/performance.html#Caches-Prepared-Statement
 func DryRun(db *gorm.DB) {

diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange_test.go
@@ -254,9 +254,7 @@ func TestBlobberCore_DeleteFile(t *testing.T) {
 
 			config.Configuration.MaxAllocationDirFiles = tc.maxDirFilesPerAlloc
 
-			ctx := context.TODO()
-			db := datastore.GetStore().GetDB().Begin()
-			ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db)
+			ctx := datastore.GetStore().CreateTransaction(context.TODO())
 
 			change := &DeleteFileChange{
 				AllocationID: tc.allocationID,

diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go
@@ -1,10 +1,12 @@
 package allocation
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"time"
 
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
 	"github.com/0chain/blobber/code/go/0chain.net/core/common"
 	"gorm.io/gorm/clause"
 
@@ -162,7 +164,8 @@ func (*Pending) TableName() string {
 }
 
 // GetPendingWrite Get write size that is not yet redeemed
-func GetPendingWrite(db *gorm.DB, clientID, allocationID string) (pendingWriteSize int64, err error) {
+func GetPendingWrite(ctx context.Context, clientID, allocationID string) (pendingWriteSize int64, err error) {
+	db := datastore.GetStore().GetTransaction(ctx)
 	err = db.Model(&Pending{}).Select("pending_write").Where(
 		"id=?", fmt.Sprintf("%v:%v", clientID, allocationID),
 	).Scan(&pendingWriteSize).Error
@@ -177,7 +180,8 @@ func GetPendingWrite(db *gorm.DB, clientID, allocationID string) (pendingWriteSi
 }
 
 // GetPendingRead Get read size that is not yet redeemed
-func GetPendingRead(db *gorm.DB, clientID, allocationID string) (pendingReadSize int64, err error) {
+func GetPendingRead(ctx context.Context, clientID, allocationID string) (pendingReadSize int64, err error) {
+	db := datastore.GetStore().GetTransaction(ctx)
 	err = db.Model(&Pending{}).Select("pending_read").Where(
 		"id=?", fmt.Sprintf("%v:%v", clientID, allocationID),
 	).Scan(&pendingReadSize).Error
@@ -191,7 +195,8 @@ func GetPendingRead(db *gorm.DB, clientID, allocationID string) (pendingReadSize
 	return
 }
 
-func AddToPending(db *gorm.DB, clientID, allocationID string, pendingWrite int64) (err error) {
+func AddToPending(ctx context.Context, clientID, allocationID string, pendingWrite int64) (err error) {
+	db := datastore.GetStore().GetTransaction(ctx)
 	key := clientID + ":" + allocationID
 	// Lock is required because two process can simultaneously call this function and read pending data
 	// thus giving same value leading to inconsistent data
@@ -215,7 +220,8 @@ func AddToPending(db *gorm.DB, clientID, allocationID string, pendingWrite int64
 	return nil
 }
 
-func GetWritePoolsBalance(db *gorm.DB, allocationID string) (balance uint64, err error) {
+func GetWritePoolsBalance(ctx context.Context, allocationID string) (balance uint64, err error) {
+	db := datastore.GetStore().GetTransaction(ctx)
 	err = db.Model(&WritePool{}).Select("COALESCE(SUM(balance),0) as tot_balance").Where(
 		"allocation_id = ?", allocationID,
 	).Scan(&balance).Error
@@ -266,36 +272,39 @@ func (WritePool) TableName() string {
 	return "write_pools"
 }
 
-func GetReadPool(db *gorm.DB, clientID string) (*ReadPool, error) {
+func GetReadPool(ctx context.Context, clientID string) (*ReadPool, error) {
+	db := datastore.GetStore().GetTransaction(ctx)
 	var rp ReadPool
 	return &rp, db.Model(&ReadPool{}).Where("client_id = ?", clientID).Scan(&rp).Error
 }
 
-func GetReadPoolsBalance(db *gorm.DB, clientID string) (int64, error) {
-	rp, err := GetReadPool(db, clientID)
+func GetReadPoolsBalance(ctx context.Context, clientID string) (int64, error) {
+	rp, err := GetReadPool(ctx, clientID)
 	if err != nil {
 		return 0, err
 	}
 
 	return rp.Balance, nil
 }
 
-func UpsertReadPool(db *gorm.DB, rp *ReadPool) error {
+func UpsertReadPool(ctx context.Context, rp *ReadPool) error {
 	updateFields := []string{"balance"}
-
+	db := datastore.GetStore().GetTransaction(ctx)
 	return db.Clauses(clause.OnConflict{
 		Columns:   []clause.Column{{Name: "client_id"}},
 		DoUpdates: clause.AssignmentColumns(updateFields), // column needed to be updated
 	}).Create(&rp).Error
 }
 
-func UpdateReadPool(db *gorm.DB, rp *ReadPool) error {
+func UpdateReadPool(ctx context.Context, rp *ReadPool) error {
+	db := datastore.GetStore().GetTransaction(ctx)
 	return db.Model(&ReadPool{}).Where("client_id = ?", rp.ClientID).Updates(map[string]interface{}{
 		"balance": rp.Balance,
 	}).Error
 }
 
-func SetWritePool(db *gorm.DB, allocationID string, wp *WritePool) (err error) {
+func SetWritePool(ctx context.Context, allocationID string, wp *WritePool) (err error) {
+	db := datastore.GetStore().GetTransaction(ctx)
 	err = db.Delete(&WritePool{}, "allocation_id = ?", allocationID).Error
 	if err != nil {
 		return

diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go
@@ -95,9 +95,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) {
 
 			config.Configuration.MaxAllocationDirFiles = tc.maxDirFilesPerAlloc
 
-			ctx := context.TODO()
-			db := datastore.GetStore().GetDB().Begin()
-			ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db)
+			ctx := datastore.GetStore().CreateTransaction(context.TODO())
 
 			fPath := "/new"
 			change := &UploadFileChanger{

diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/movefilechange_test.go
@@ -468,9 +468,7 @@ func TestBlobberCore_MoveFile(t *testing.T) {
 
 			config.Configuration.MaxAllocationDirFiles = tc.maxDirFilesPerAlloc
 
-			ctx := context.TODO()
-			db := datastore.GetStore().GetDB().Begin()
-			ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db)
+			ctx := datastore.GetStore().CreateTransaction(context.TODO())
 
 			change := &MoveFileChange{
 				AllocationID: tc.allocationID,

diff --git a/code/go/0chain.net/blobbercore/allocation/multiop_test.go b/code/go/0chain.net/blobbercore/allocation/multiop_test.go
@@ -32,9 +32,7 @@ func TestMultiOp(t *testing.T) {
 	alloc.OwnerPublicKey = sch.GetPublicKey()
 	alloc.OwnerID = client.GetClientID()
 	datastore.MocketTheStore(t, true)
-	ctx := context.TODO()
-	db := datastore.GetStore().GetDB().Begin()
-	ctx = context.WithValue(ctx, datastore.ContextKeyTransaction, db)
+	ctx := datastore.GetStore().CreateTransaction(context.TODO())
 	setupDbMock()
 	fileIDMeta := make(map[string]string)
 	fileIDMeta["/"] = randName()