Merge branch 'sprint-july-3' into fix/remove-postgres-post

.github/workflows/build-&-publish-docker-image.yml

@@ -16,6 +16,8 @@ env:
   BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }}
   VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }}
   DOCKER_CLI_EXPERIMENTAL: enabled
+  BLOBBER_BUILDBASE: blobber_base
+  BLOBBER_BUILD_BASE_REGISTRY: ${{ secrets.BLOBBER_BUILD_BASE_REGISTRY }}
 
 jobs:
   blobber:
@@ -39,7 +41,7 @@ jobs:
           go-version: ^1.20 # The Go version to download (if necessary) and use.
 
       - name: Clone blobber
-        uses: actions/checkout@v1
+        uses: actions/checkout@v3
 
       - name: Set up Docker Buildx
         run: |
@@ -60,6 +62,26 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
+      - name: Get changed files using defaults
+        id: changed-files
+        uses: tj-actions/changed-files@v18.4
+
+      - name: Pull Build Base
+        run: |
+          docker pull $BLOBBER_BUILD_BASE_REGISTRY:staging
+          docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE
+
+      - name: Build Base image
+        if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile')
+        run: |
+          SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
+
+          ./docker.local/bin/build.base.sh &&
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+
       - name: Build blobber
         run: |
           SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
@@ -68,7 +90,7 @@ jobs:
           export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
           export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:${TAG} -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}"
           docker buildx create --driver-opt network=host --use --buildkitd-flags '--allow-insecure-entitlement security.insecure' --use blobber_buildx
-          ./docker.local/bin/build.base.sh && ./docker.local/bin/build.blobber.sh
+          ./docker.local/bin/build.blobber.sh
 
   validator:
     runs-on: [self-hosted, arc-runner]
@@ -112,14 +134,34 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
+      - name: Get changed files using defaults
+        id: changed-files
+        uses: tj-actions/changed-files@v18.4
+
+      - name: Pull Build Base
+        run: |
+          docker pull $BLOBBER_BUILD_BASE_REGISTRY:staging
+          docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE
+
+      - name: Build Base image
+        if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile')
+        run: |
+          SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
+
+          ./docker.local/bin/build.base.sh 
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG
+          docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA
+
       - name: Build validator
         run: |
           SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
           export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base"
           export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
           export DOCKER_IMAGE_VALIDATOR="-t ${VALIDATOR_REGISTRY}:${TAG} -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}"
           docker buildx create --driver-opt network=host --use --buildkitd-flags '--allow-insecure-entitlement security.insecure' --use blobber_buildx
-          ./docker.local/bin/build.base.sh && ./docker.local/bin/build.validator.sh
+          ./docker.local/bin/build.validator.sh
 
 
   system-tests:

code/go/0chain.net/blobbercore/challenge/challenge.go

@@ -24,25 +24,22 @@ type BCChallengeResponse struct {
 	Challenges []*ChallengeEntity `json:"challenges"`
 }
 
-var lastChallengeTimestamp int
+var lastChallengeRound int64
 
 func syncOpenChallenges(ctx context.Context) {
-	const incrOffset = 20
 	defer func() {
 		if r := recover(); r != nil {
 			logging.Logger.Error("[recover]challenge", zap.Any("err", r))
 		}
 	}()
 
-	offset := 0
 	params := make(map[string]string)
 	params["blobber"] = node.Self.ID
-	params["offset"] = strconv.Itoa(offset)
-	params["limit"] = "20"
-	if lastChallengeTimestamp > 0 {
-		params["from"] = strconv.Itoa(lastChallengeTimestamp)
+
+	params["limit"] = "50"
+	if lastChallengeRound > 0 {
+		params["from"] = strconv.FormatInt(lastChallengeRound, 10)
 	}
-	logging.Logger.Info("[challenge]sync:pull", zap.Any("params", params))
 	start := time.Now()
 
 	var downloadElapsed, jsonElapsed time.Duration
@@ -54,6 +51,9 @@ func syncOpenChallenges(ctx context.Context) {
 			return
 		default:
 		}
+
+		logging.Logger.Info("[challenge]sync:pull", zap.Any("params", params))
+
 		var challenges BCChallengeResponse
 		var challengeIDs []string
 		challenges.Challenges = make([]*ChallengeEntity, 0)
@@ -75,13 +75,13 @@ func syncOpenChallenges(ctx context.Context) {
 			break
 		}
 		sort.Slice(challenges.Challenges, func(i, j int) bool {
-			return challenges.Challenges[i].CreatedAt < challenges.Challenges[j].CreatedAt
+			return challenges.Challenges[i].RoundCreatedAt < challenges.Challenges[j].RoundCreatedAt
 		})
 		count += len(challenges.Challenges)
 		for _, c := range challenges.Challenges {
 			challengeIDs = append(challengeIDs, c.ChallengeID)
-			if c.CreatedAt > common.Timestamp(lastChallengeTimestamp) {
-				lastChallengeTimestamp = int(c.CreatedAt)
+			if c.RoundCreatedAt >= lastChallengeRound {
+				lastChallengeRound = c.RoundCreatedAt
 			}
 			toProcessChallenge <- c
 		}
@@ -93,8 +93,6 @@ func syncOpenChallenges(ctx context.Context) {
 		if len(challenges.Challenges) == 0 {
 			break
 		}
-		offset += incrOffset
-		params["offset"] = strconv.Itoa(offset)
 	}
 
 	dbTimeStart := time.Now()
@@ -110,6 +108,11 @@ func syncOpenChallenges(ctx context.Context) {
 
 func validateOnValidators(c *ChallengeEntity) {
 
+	logging.Logger.Info("[challenge]validate: ",
+		zap.Any("challenge", c),
+		zap.String("challenge_id", c.ChallengeID),
+	)
+
 	ctx := datastore.GetStore().CreateTransaction(context.TODO())
 	defer ctx.Done()
 
@@ -119,7 +122,7 @@ func validateOnValidators(c *ChallengeEntity) {
 		logging.Logger.Error("[challengetiming]add: ",
 			zap.String("challenge_id", c.ChallengeID),
 			zap.Error(err))
-		deleteChallenge(int64(c.CreatedAt))
+		deleteChallenge(c.RoundCreatedAt)
 		tx.Rollback()
 	}
 
@@ -150,7 +153,7 @@ func validateOnValidators(c *ChallengeEntity) {
 			zap.Time("created", createdTime),
 			zap.Error(err))
 		//TODO: Should we delete the challenge from map or send it back to the todo channel?
-		deleteChallenge(int64(c.CreatedAt))
+		deleteChallenge(c.RoundCreatedAt)
 		tx.Rollback()
 		return
 	}

code/go/0chain.net/blobbercore/challenge/entity.go

@@ -97,8 +97,9 @@ type ChallengeEntity struct {
 	Timestamp               common.Timestamp      `gorm:"column:timestamp;not null;default:0" json:"timestamp"`
 
 	// This time is taken from Blockchain challenge object.
-	CreatedAt common.Timestamp `gorm:"created_at" json:"created"`
-	UpdatedAt time.Time        `gorm:"updated_at;type:timestamp without time zone;not null;default:current_timestamp" json:"-"`
+	RoundCreatedAt int64            `gorm:"round_created_at" json:"round_created_at"`
+	CreatedAt      common.Timestamp `gorm:"created_at" json:"created"`
+	UpdatedAt      time.Time        `gorm:"updated_at;type:timestamp without time zone;not null;default:current_timestamp" json:"-"`
 }
 
 func (ChallengeEntity) TableName() string {

code/go/0chain.net/blobbercore/challenge/protocol.go

@@ -48,7 +48,7 @@ type ChallengeResponse struct {
 func (cr *ChallengeEntity) CancelChallenge(ctx context.Context, errReason error) {
 	cancellation := time.Now()
 	db := datastore.GetStore().GetTransaction(ctx)
-	deleteChallenge(int64(cr.CreatedAt))
+	deleteChallenge(cr.RoundCreatedAt)
 	cr.Status = Cancelled
 	cr.StatusMessage = errReason.Error()
 	cr.UpdatedAt = cancellation.UTC()

code/go/0chain.net/blobbercore/challenge/worker.go

@@ -81,6 +81,8 @@ func challengeProcessor(ctx context.Context) {
 			return
 
 		case it := <-toProcessChallenge:
+
+			logging.Logger.Info("processing_challenge", zap.Any("challenge_id", it.ChallengeID))
 			if ok := it.createChallenge(); !ok {
 				continue
 			}
@@ -142,7 +144,7 @@ func commitOnChainWorker(ctx context.Context) {
 					}()
 					err := challenge.VerifyChallengeTransaction(txn)
 					if err == nil || err != ErrEntityNotFound {
-						deleteChallenge(int64(challenge.CreatedAt))
+						deleteChallenge(int64(challenge.RoundCreatedAt))
 					}
 				}(&chall)
 			}
@@ -175,11 +177,11 @@ func getBatch(batchSize int) (chall []ChallengeEntity) {
 
 func (it *ChallengeEntity) createChallenge() bool {
 	challengeMapLock.Lock()
-	if _, ok := challengeMap.Get(int64(it.CreatedAt)); ok {
+	if _, ok := challengeMap.Get(it.RoundCreatedAt); ok {
 		challengeMapLock.Unlock()
 		return false
 	}
-	challengeMap.Put(int64(it.CreatedAt), it)
+	challengeMap.Put(it.RoundCreatedAt, it)
 	challengeMapLock.Unlock()
 	return true
 }

docker.local/.dockerignore

@@ -1,8 +1,13 @@
 passphrase.txt
 logs/
+docs/
 .git
 *.md
 .cache
 awsnet
 docker.local/blobber*
+docker.aws/*
+config/
+docker-clean/
+keys_config
 **/pkg

go.mod

@@ -21,12 +21,12 @@ require (
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
 	go.uber.org/zap v1.24.0
-	golang.org/x/crypto v0.10.0
+	golang.org/x/crypto v0.11.0
 	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.9.0
+	golang.org/x/sys v0.10.0
 	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect
-	google.golang.org/grpc v1.56.1
+	google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e //  indirect
+	google.golang.org/grpc v1.56.2
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
@@ -44,7 +44,7 @@ require (
 
 require google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
 
-require github.com/pressly/goose/v3 v3.13.1
+require github.com/pressly/goose/v3 v3.13.4
 
 require (
 	github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 // indirect
@@ -85,11 +85,11 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/pgx/v5 v5.4.0 // indirect
+	github.com/jackc/pgx/v5 v5.4.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.16.6 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/klauspost/reedsolomon v1.11.7 // indirect
 	github.com/machinebox/graphql v0.2.2 // indirect
@@ -120,7 +120,7 @@ require (
 	go.mongodb.org/mongo-driver v1.11.3 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
-	golang.org/x/text v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect