From 54b3c2a08d4edbfafcaae987e8035e481f880fb3 Mon Sep 17 00:00:00 2001 From: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Date: Sun, 16 Jul 2023 20:41:30 +0530 Subject: [PATCH 1/4] Sprint july 2 (#1177) * Add path in thumbnail hash (#1098) * get path from changes * pass rootRef in apply changes * add logs * fix deletechange * fix multi op and copy * fix delete root dir * fix delete root change * fix lint issue * cleanup and stats change * rename func to processMove * calculate object tree in ref path * rmv return from deleteChange * revert changes * move to filestore changes * check prevRoot * add logs and condition * check filestore * requested changes * validate nil proof * add move and copy dir tests * add logs * use path to add child * fix error log * fix add child * add proof log * add more logs * change read log * log objectPath * add commit logs * commit thumbnail first * add missing param * add missing param * add path in the thumbnail hash * add allocID and validation root index * use write * add condn in count query * rmv thumbnail_filename * log wm * cleanup * add rollback wm check * allow empty allocation root * fix tests * check path in upload * fix check * empty commit --------- Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Co-authored-by: Yury * Dep/update (#1153) * dependencies update * dependencies update * Remove fileID from fileMetaHash (#1114) * rmv fileID from hash calc * Trigger Build * adding false commit to restart systemtests * add path in fileMetaHash * empty commit --------- Co-authored-by: Yury Co-authored-by: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> Co-authored-by: boddumanohar Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> * optimize image (#1148) Co-authored-by: boddumanohar * once for logger init * Update challenge timing submission (#1140) * update challenge timing submission * fix createdAt in challenge timing table --------- Co-authored-by: Yury * Fix blobber size (#1163) * Do not redeem readmarkers for free reads (#1166) * do not redeem readmarkers for free reads * fix unit tests, remove readmarker handling from download method * once for logger init (#1156) * once for logger init * init logging the same way we do in 0chain * init logging the same way we do in 0chain * Hotfix/map concurrent write (#1158) * once for logger init * wider locking window * Hotfix/remove custom nonce (#1141) * removed custom nonce managing logic * fixed logging * updated gosdk * uncommented previous nonce logic * uncommented previous nonce logic * uncommented previous nonce logic * uncommented previous nonce logic * Use single file to avoid maintaining multiple version of same file (#1160) * Use single file to avoid maintaining multiple version of same file * Fix config path * Remove variable * Update path variable for blobber service --------- Co-authored-by: Yury * fix consume quota lock (#1173) * Fix rm and wm timestamp (#1162) * once for logger init * init logging the same way we do in 0chain * init logging the same way we do in 0chain * add lastChallengeTime log * empty commit * refactor timestamp check in markers * Add diff txn for move to filestore * return if duplicate --------- Co-authored-by: dabasov * fixed gitactions fix issue * remove printing private key in logs (#1161) Co-authored-by: Yury * Fix concurrent upload issue (#1174) * Fix concurrent upload issue * Fix concurrent upload issue * Fix/concurrent upload issue (#1175) * Fix concurrent upload issue * Fix concurrent upload issue * Fix concurrent upload issue * Fix concurrent upload issue --------- Co-authored-by: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Co-authored-by: Yury Co-authored-by: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> Co-authored-by: boddumanohar Co-authored-by: Manali-Jain-squareops <86873900+Manali-Jain-squareops@users.noreply.github.com> Co-authored-by: Jayash Satolia <73050737+Jayashsatolia403@users.noreply.github.com> Co-authored-by: stewartie4 Co-authored-by: Dinmukhammed <52813950+din-mukhammed@users.noreply.github.com> Co-authored-by: Laxmi Prasad Oli Co-authored-by: root --- .../workflows/build-for-conductor-testing.yml | 2 +- code/go/0chain.net/blobber/logging.go | 2 +- code/go/0chain.net/blobber/node.go | 3 +- .../allocation/allocationchange.go | 40 +- .../blobbercore/allocation/entity.go | 12 + .../allocation/file_changer_base.go | 21 +- .../allocation/file_changer_update.go | 1 - .../allocation/file_changer_upload.go | 1 - .../blobbercore/allocation/protocol.go | 17 +- .../0chain.net/blobbercore/allocation/zcn.go | 8 +- .../blobbercore/challenge/challenge.go | 4 +- .../blobbercore/challenge/protocol.go | 43 +- .../blobbercore/filestore/storage.go | 36 +- .../blobbercore/filestore/store_test.go | 101 +++-- .../blobbercore/filestore/tree_validation.go | 9 +- .../blobbercore/handler/download_quota.go | 4 +- .../handler/download_request_header.go | 2 - .../handler/file_command_update.go | 1 + .../handler/file_command_upload.go | 4 + .../handler/handler_download_test.go | 419 ++++++------------ .../handler/object_operation_handler.go | 124 ++---- .../handler/object_operation_handler_test.go | 97 ++-- .../blobbercore/readmarker/readmarker.go | 8 +- .../blobbercore/reference/object.go | 1 + .../0chain.net/blobbercore/reference/ref.go | 17 +- .../blobbercore/writemarker/protocol.go | 10 + .../blobbercore/writemarker/worker.go | 2 +- code/go/0chain.net/core/logging/logger.go | 52 ++- code/go/0chain.net/core/transaction/nonce.go | 25 +- code/go/0chain.net/validator/main.go | 1 - .../storage/challenge_handler.go | 5 - .../storage/challenge_handler_test.go | 10 - .../validatorcore/storage/models.go | 29 +- docker.local/.dockerignore | 5 +- docker.local/b0docker-compose.yml | 4 +- docker.local/conductor-b0docker-compose.yml | 89 ---- go.mod | 23 +- go.sum | 60 ++- 38 files changed, 530 insertions(+), 762 deletions(-) delete mode 100644 docker.local/conductor-b0docker-compose.yml diff --git a/.github/workflows/build-for-conductor-testing.yml b/.github/workflows/build-for-conductor-testing.yml index 59301c67c..a77432ca0 100644 --- a/.github/workflows/build-for-conductor-testing.yml +++ b/.github/workflows/build-for-conductor-testing.yml @@ -1,7 +1,7 @@ name: "Build Docker Image for conductor testing" concurrency: - group: "publish-${{ github.ref }}" + group: "conductor-${{ github.ref }}" cancel-in-progress: true on: diff --git a/code/go/0chain.net/blobber/logging.go b/code/go/0chain.net/blobber/logging.go index 017bdd0f3..339e8807c 100644 --- a/code/go/0chain.net/blobber/logging.go +++ b/code/go/0chain.net/blobber/logging.go @@ -17,7 +17,7 @@ func setupLogging() { logging.InitLogging("production", logDir, "0chainBlobber.log") } - zcncore.SetLogFile(logDir+"/0chainBlobber.log", false) + zcncore.SetLogFile(logDir+"/0chainBlobber-sdk.log", false) zcncore.SetLogLevel(3) fmt.Print(" [OK]\n") } diff --git a/code/go/0chain.net/blobber/node.go b/code/go/0chain.net/blobber/node.go index 71f588a39..1da96a289 100644 --- a/code/go/0chain.net/blobber/node.go +++ b/code/go/0chain.net/blobber/node.go @@ -39,10 +39,9 @@ func setupNode() error { } } - fmt.Println("*== Validator Wallet Info ==*") + fmt.Println("*== Blobber Wallet Info ==*") fmt.Println(" ID: ", node.Self.ID) fmt.Println(" Public Key: ", publicKey) - fmt.Println(" Private Key: ", privateKey) fmt.Println("*===========================*") logging.Logger.Info(" Base URL" + node.Self.GetURLBase()) diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index cee994450..703009b63 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -250,25 +250,27 @@ type Result struct { } // TODO: Need to speed up this function -func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { +func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) (err error) { logging.Logger.Info("Move to filestore", zap.String("allocation_id", a.AllocationID)) - tx := datastore.GetStore().GetTransaction(ctx) + tx := datastore.GetStore().GetDB().Begin() var refs []*Result limitCh := make(chan struct{}, 10) wg := &sync.WaitGroup{} - err := tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "validation_root", "thumbnail_hash", "prev_validation_root", "prev_thumbnail_hash").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). + err = tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "validation_root", "thumbnail_hash", "prev_validation_root", "prev_thumbnail_hash").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). FindInBatches(&refs, 50, func(tx *gorm.DB, batch int) error { for _, ref := range refs { var count int64 - tx.Model(&reference.Ref{}). - Where("allocation_id=? AND validation_root=? AND type=?", a.AllocationID, ref.PrevValidationRoot, reference.FILE). - Count(&count) + if ref.PrevValidationRoot != "" { + tx.Model(&reference.Ref{}). + Where("allocation_id=? AND validation_root=?", a.AllocationID, ref.PrevValidationRoot). + Count(&count) + } limitCh <- struct{}{} wg.Add(1) @@ -279,13 +281,11 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { wg.Done() }() - if count == 0 { - if ref.PrevValidationRoot != "" { - err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevValidationRoot) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), - zap.String("validation_root", ref.ValidationRoot)) - } + if count == 0 && ref.PrevValidationRoot != "" { + err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevValidationRoot) + if err != nil { + logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), + zap.String("validation_root", ref.ValidationRoot)) } } err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.ValidationRoot) @@ -319,20 +319,23 @@ func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { if err != nil { logging.Logger.Error("Error while moving to filestore", zap.Error(err)) + tx.Rollback() return err } err = tx.Exec("UPDATE reference_objects SET is_precommit=?, prev_validation_root=validation_root, prev_thumbnail_hash=thumbnail_hash WHERE allocation_id=? AND is_precommit=? AND deleted_at is NULL", false, a.AllocationID, true).Error if err != nil { + tx.Rollback() return err } + tx.Commit() return deleteFromFileStore(ctx, a.AllocationID) } func deleteFromFileStore(ctx context.Context, allocationID string) error { - db := datastore.GetStore().GetTransaction(ctx) + db := datastore.GetStore().GetDB().Begin() limitCh := make(chan struct{}, 10) wg := &sync.WaitGroup{} var results []Result @@ -385,13 +388,20 @@ func deleteFromFileStore(ctx context.Context, allocationID string) error { wg.Wait() if err != nil && err != gorm.ErrRecordNotFound { logging.Logger.Error("DeleteFromFileStore", zap.Error(err)) + db.Rollback() return err } - return db.Model(&reference.Ref{}).Unscoped(). + err = db.Model(&reference.Ref{}).Unscoped(). Delete(&reference.Ref{}, "allocation_id = ? AND deleted_at IS NOT NULL", allocationID).Error + if err != nil { + db.Rollback() + return err + } + db.Commit() + return nil } // Note: We are also fetching refPath for srcPath in copy operation diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go index b0d412518..d4d953d98 100644 --- a/code/go/0chain.net/blobbercore/allocation/entity.go +++ b/code/go/0chain.net/blobbercore/allocation/entity.go @@ -139,6 +139,18 @@ func (a *Allocation) GetRequiredWriteBalance(blobberID string, writeSize int64, return } +// IsReadFree Determine if read price is 0 +func (a *Allocation) IsReadFree(blobberID string) bool { + for _, d := range a.Terms { + if d.BlobberID == blobberID { + if d.ReadPrice == 0 { + return true + } + } + } + return false +} + type Pending struct { // ID of format client_id:allocation_id ID string `gorm:"column:id;primaryKey"` diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index 64d7d5195..698b59974 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -74,16 +74,6 @@ func (fc *BaseFileChanger) DeleteTempFile() error { func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context) error { - fileInputData := &filestore.FileInputData{} - fileInputData.Name = fc.Filename - fileInputData.Path = fc.Path - fileInputData.ValidationRoot = fc.ValidationRoot - fileInputData.FixedMerkleRoot = fc.FixedMerkleRoot - fileInputData.ChunkSize = fc.ChunkSize - _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData) - if err != nil { - return common.NewError("file_store_error", "Error committing to file store. "+err.Error()) - } if fc.ThumbnailSize > 0 { fileInputData := &filestore.FileInputData{} fileInputData.Name = fc.ThumbnailFilename @@ -96,6 +86,15 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context) error { return common.NewError("file_store_error", "Error committing thumbnail to file store. "+err.Error()) } } - + fileInputData := &filestore.FileInputData{} + fileInputData.Name = fc.Filename + fileInputData.Path = fc.Path + fileInputData.ValidationRoot = fc.ValidationRoot + fileInputData.FixedMerkleRoot = fc.FixedMerkleRoot + fileInputData.ChunkSize = fc.ChunkSize + _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData) + if err != nil { + return common.NewError("file_store_error", "Error committing to file store. "+err.Error()) + } return nil } diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index 9d720ddb4..97cbdf7e0 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -97,7 +97,6 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference fileRef.EncryptedKey = nf.EncryptedKey fileRef.ChunkSize = nf.ChunkSize fileRef.IsPrecommit = true - fileRef.ThumbnailFilename = nf.ThumbnailFilename return rootRef, nil } diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 0fbf5925a..c45bb33cf 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -103,7 +103,6 @@ func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference UpdatedAt: ts, HashToBeComputed: true, IsPrecommit: true, - ThumbnailFilename: nf.ThumbnailFilename, } fileID, ok := fileIDMeta[newFile.Path] diff --git a/code/go/0chain.net/blobbercore/allocation/protocol.go b/code/go/0chain.net/blobbercore/allocation/protocol.go index 25abfebb2..a50b3cbf8 100644 --- a/code/go/0chain.net/blobbercore/allocation/protocol.go +++ b/code/go/0chain.net/blobbercore/allocation/protocol.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" @@ -103,8 +104,7 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc } foundBlobber = true a.AllocationRoot = "" - a.BlobberSize = (sa.Size + int64(len(sa.BlobberDetails)-1)) / - int64(len(sa.BlobberDetails)) + a.BlobberSize = int64(math.Ceil(float64(sa.Size) / float64(sa.DataShards))) a.BlobberSizeUsed = 0 break } @@ -154,23 +154,14 @@ func FetchAllocationFromEventsDB(ctx context.Context, allocationID string, alloc logging.Logger.Info("Saving the allocation to DB") - if isExist { - err = tx.Save(a).Error - } else { - err = tx.Create(a).Error - } - + err = tx.Save(a).Error if err != nil { return nil, err } // save/update related terms for _, t := range a.Terms { - if isExist { - err = tx.Save(t).Error - } else { - err = tx.Create(t).Error - } + err = tx.Save(t).Error if err != nil { return nil, err } diff --git a/code/go/0chain.net/blobbercore/allocation/zcn.go b/code/go/0chain.net/blobbercore/allocation/zcn.go index f6bb744ab..0dff85ceb 100644 --- a/code/go/0chain.net/blobbercore/allocation/zcn.go +++ b/code/go/0chain.net/blobbercore/allocation/zcn.go @@ -6,6 +6,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/errors" "gorm.io/gorm" + "math" ) // SyncAllocation try to pull allocation from blockchain, and insert it in db. @@ -24,8 +25,7 @@ func SyncAllocation(allocationId string) (*Allocation, error) { belongToThisBlobber = true alloc.AllocationRoot = "" - alloc.BlobberSize = (sa.Size + int64(len(sa.BlobberDetails)-1)) / - int64(len(sa.BlobberDetails)) + alloc.BlobberSize = int64(math.Ceil(float64(sa.Size) / float64(sa.DataShards))) alloc.BlobberSizeUsed = 0 break @@ -62,12 +62,12 @@ func SyncAllocation(allocationId string) (*Allocation, error) { } err = datastore.GetStore().GetDB().Transaction(func(tx *gorm.DB) error { - if err := tx.Table(TableNameAllocation).Create(alloc).Error; err != nil { + if err := tx.Table(TableNameAllocation).Save(alloc).Error; err != nil { return err } for _, term := range terms { - if err := tx.Table(TableNameTerms).Create(term).Error; err != nil { + if err := tx.Table(TableNameTerms).Save(term).Error; err != nil { return err } } diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go index 9fed39dad..3f77cb801 100644 --- a/code/go/0chain.net/blobbercore/challenge/challenge.go +++ b/code/go/0chain.net/blobbercore/challenge/challenge.go @@ -42,6 +42,7 @@ func syncOpenChallenges(ctx context.Context) { if lastChallengeTimestamp > 0 { params["from"] = strconv.Itoa(lastChallengeTimestamp) } + logging.Logger.Info("[challenge]sync:pull", zap.Any("params", params)) start := time.Now() var downloadElapsed, jsonElapsed time.Duration @@ -118,6 +119,8 @@ func validateOnValidators(c *ChallengeEntity) { logging.Logger.Error("[challengetiming]add: ", zap.String("challenge_id", c.ChallengeID), zap.Error(err)) + deleteChallenge(int64(c.CreatedAt)) + tx.Rollback() } createdTime := common.ToTime(c.CreatedAt) @@ -246,5 +249,4 @@ func (c *ChallengeEntity) getCommitTransaction() (*transaction.Transaction, erro } return txn, nil - } diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index 246230053..1c297fdf3 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -22,6 +22,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/util" sdkUtil "github.com/0chain/gosdk/core/util" "github.com/remeh/sizedwaitgroup" + "gorm.io/gorm" "go.uber.org/zap" ) @@ -96,35 +97,40 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { } rootRef, err := reference.GetReference(ctx, cr.AllocationID, "/") - if err != nil { + if err != nil && err != gorm.ErrRecordNotFound { allocMu.RUnlock() cr.CancelChallenge(ctx, err) return err } blockNum := int64(0) - if rootRef.NumBlocks > 0 { - r := rand.New(rand.NewSource(cr.RandomNumber)) - blockNum = r.Int63n(rootRef.NumBlocks) - blockNum++ - cr.BlockNum = blockNum - } + var objectPath *reference.ObjectPath + if rootRef != nil { + if rootRef.NumBlocks > 0 { + r := rand.New(rand.NewSource(cr.RandomNumber)) + blockNum = r.Int63n(rootRef.NumBlocks) + blockNum++ + cr.BlockNum = blockNum + } - logging.Logger.Info("[challenge]rand: ", zap.Any("rootRef.NumBlocks", rootRef.NumBlocks), zap.Any("blockNum", blockNum), zap.Any("challenge_id", cr.ChallengeID), zap.Any("random_seed", cr.RandomNumber)) - objectPath, err := reference.GetObjectPath(ctx, cr.AllocationID, blockNum) - if err != nil { - allocMu.RUnlock() - cr.CancelChallenge(ctx, err) - return err - } + logging.Logger.Info("[challenge]rand: ", zap.Any("rootRef.NumBlocks", rootRef.NumBlocks), zap.Any("blockNum", blockNum), zap.Any("challenge_id", cr.ChallengeID), zap.Any("random_seed", cr.RandomNumber)) + objectPath, err = reference.GetObjectPath(ctx, cr.AllocationID, blockNum) + if err != nil { + allocMu.RUnlock() + cr.CancelChallenge(ctx, err) + return err + } - cr.RefID = objectPath.RefID + cr.RefID = objectPath.RefID + cr.ObjectPath = objectPath + } cr.RespondedAllocationRoot = allocationObj.AllocationRoot - cr.ObjectPath = objectPath postData := make(map[string]interface{}) postData["challenge_id"] = cr.ChallengeID - postData["object_path"] = objectPath + if objectPath != nil { + postData["object_path"] = objectPath + } markersArray := make([]map[string]interface{}, 0) for _, wm := range wms { markersMap := make(map[string]interface{}) @@ -188,6 +194,9 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { postData["challenge_proof"] = challengeResponse } + if objectPath == nil { + objectPath = &reference.ObjectPath{} + } err = UpdateChallengeTimingProofGenerationAndFileSize( cr.ChallengeID, proofGenTime, diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 060b5a137..fd90e9b4c 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -26,6 +26,7 @@ package filestore // import ( + "bytes" "context" "encoding/hex" "errors" @@ -122,7 +123,6 @@ func (fs *FileStore) MoveToFilestore(allocID, hash string) error { } _ = os.Rename(preCommitPath, fPath) - return nil } @@ -132,7 +132,7 @@ func (fs *FileStore) DeleteFromFilestore(allocID, hash string) error { if err != nil { return common.NewError("get_file_path_error", err.Error()) } - + logging.Logger.Info("Deleting file from filestore", zap.String("path", fPath)) err = os.Remove(fPath) if err != nil { return common.NewError("blob_object_dir_creation_error", err.Error()) @@ -159,7 +159,8 @@ func (fs *FileStore) DeletePreCommitDir(allocID string) error { func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) (bool, error) { - logging.Logger.Info("Committing file") + logging.Logger.Info("Committing write", zap.String("allocation_id", allocID), zap.Any("file_data", fileData)) + tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, encryption.Hash(fileData.Path), conID) fileHash := fileData.ValidationRoot @@ -181,25 +182,7 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) r, err := os.Open(tempFilePath) if err != nil { - - if errors.Is(err, os.ErrNotExist) { - f.Close() - _ = os.Remove(preCommitPath) - return true, nil - } return false, err - } else { - // check if file is empty - check_file, err := os.Stat(tempFilePath) - if err == nil && check_file.Size() == 0 { - f.Close() - _ = os.Remove(preCommitPath) - return true, nil - } else if err != nil { - f.Close() - _ = os.Remove(preCommitPath) - return false, err - } } defer f.Close() @@ -220,6 +203,10 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) if err != nil { return false, common.NewError("read_error", err.Error()) } + _, err = h.Write([]byte(fileData.Path)) + if err != nil { + return false, common.NewError("read_error", err.Error()) + } hash := hex.EncodeToString(h.Sum(nil)) if hash != fileData.ThumbnailHash { return false, common.NewError("hash_mismatch", @@ -803,6 +790,13 @@ func (fs *FileStore) updateAllocTempFileSize(allocID string, size int64) { alloc.tmpFileSize += uint64(size) } +func (fs *FileStore) pathWriter(path string) io.Reader { + + pathBytes := []byte(path) + buf := bytes.NewBuffer(pathBytes) + return buf +} + // GetTempFilesSizeOfAllocation Get total file sizes of all allocation that are not yet committed func (fs *FileStore) GetTotalTempFileSizes() (s uint64) { for _, alloc := range fs.mAllocs { diff --git a/code/go/0chain.net/blobbercore/filestore/store_test.go b/code/go/0chain.net/blobbercore/filestore/store_test.go index 8aa22a871..340da9581 100644 --- a/code/go/0chain.net/blobbercore/filestore/store_test.go +++ b/code/go/0chain.net/blobbercore/filestore/store_test.go @@ -3,6 +3,7 @@ package filestore import ( "bytes" "crypto/rand" + "encoding/base64" "encoding/hex" "errors" "fmt" @@ -223,21 +224,21 @@ func TestStoreStorageWriteAndCommit(t *testing.T) { shouldCommit: true, expectedErrorOnCommit: false, }, - // { - // testName: "Should fail", - // allocID: randString(64), - // connID: randString(64), - // fileName: randString(5), - // remotePath: filepath.Join("/", randString(5)+".txt"), - // alloc: &allocation{ - // mu: &sync.Mutex{}, - // tmpMU: &sync.Mutex{}, - // }, - - // differentHash: true, - // shouldCommit: true, - // expectedErrorOnCommit: true, - // }, + { + testName: "Should fail", + allocID: randString(64), + connID: randString(64), + fileName: randString(5), + remotePath: filepath.Join("/", randString(5)+".txt"), + alloc: &allocation{ + mu: &sync.Mutex{}, + tmpMU: &sync.Mutex{}, + }, + + differentHash: true, + shouldCommit: true, + expectedErrorOnCommit: true, + }, } for _, test := range tests { @@ -302,15 +303,6 @@ func TestStoreStorageWriteAndCommit(t *testing.T) { check_file, err := os.Stat(finalPath) require.Nil(t, err) require.True(t, check_file.Size() > tF.Size()) - success, err = fs.CommitWrite(test.allocID, test.connID, fid) - require.Nil(t, err) - require.True(t, success) - _, err = os.Stat(preCommitPath) - require.NotNil(t, err) - require.ErrorContains(t, err, "no such file or directory") - check_file, err = os.Stat(finalPath) - require.Nil(t, err) - require.True(t, check_file.Size() > tF.Size()) } }) } @@ -489,6 +481,9 @@ func TestStorageUploadUpdate(t *testing.T) { h := sha3.New256() _, err = io.Copy(h, f) require.Nil(t, err) + pathWriter := fs.pathWriter(fid.Path) + _, err = io.Copy(h, pathWriter) + require.Nil(t, err) f.Close() fid.ThumbnailHash = hex.EncodeToString(h.Sum(nil)) prevThumbHash := fid.ThumbnailHash @@ -500,7 +495,6 @@ func TestStorageUploadUpdate(t *testing.T) { // Commit thumbnail file to pre-commit location success, err = fs.CommitWrite(allocID, connID, fid) - require.Nil(t, err) require.True(t, success) // Get the path of the pre-commit location of the thumbnail file and check if the file exists @@ -526,11 +520,6 @@ func TestStorageUploadUpdate(t *testing.T) { f.Close() // check if thumbnail file is written to temp location tempFilePath = fs.getTempPathForFile(allocID, thumbFileName, pathHash, connID) - - finfo, err = os.Stat(tempFilePath) - require.Nil(t, err) - require.Equal(t, finfo.Size(), int64(size)) - // Check if the hash of the thumbnail file is same as the hash of the updated thumbnail file f, err = os.Open(tempFilePath) require.Nil(t, err) @@ -538,6 +527,9 @@ func TestStorageUploadUpdate(t *testing.T) { h = sha3.New256() _, err = io.Copy(h, f) require.Nil(t, err) + pathWriter = fs.pathWriter(fid.Path) + _, err = io.Copy(h, pathWriter) + require.Nil(t, err) f.Close() fid.ThumbnailHash = hex.EncodeToString(h.Sum(nil)) fid.IsThumbnail = false @@ -547,17 +539,11 @@ func TestStorageUploadUpdate(t *testing.T) { err = fs.MoveToFilestore(allocID, prevThumbHash) require.Nil(t, err) - // Empty Commit should do nothing - success, err = fs.CommitWrite(allocID, connID, fid) - require.Nil(t, err) - require.True(t, success) - // Set fields to commit thumbnail file fid.IsThumbnail = true fid.Name = thumbFileName // Commit thumbnail file to pre-commit location success, err = fs.CommitWrite(allocID, connID, fid) - require.Nil(t, err) require.True(t, success) @@ -576,6 +562,9 @@ func TestStorageUploadUpdate(t *testing.T) { h = sha3.New256() _, err = io.Copy(h, preFile) require.Nil(t, err) + pathWriter = fs.pathWriter(fid.Path) + _, err = io.Copy(h, pathWriter) + require.Nil(t, err) require.Equal(t, hex.EncodeToString(h.Sum(nil)), fid.ThumbnailHash) input := &ReadBlockInput{ @@ -594,6 +583,9 @@ func TestStorageUploadUpdate(t *testing.T) { buf := bytes.NewReader(data.Data) _, err = io.Copy(h, buf) require.Nil(t, err) + pathWriter = fs.pathWriter(fid.Path) + _, err = io.Copy(h, pathWriter) + require.Nil(t, err) require.Equal(t, hex.EncodeToString(h.Sum(nil)), fid.ThumbnailHash) fPath, err = fs.GetPathForFile(allocID, prevThumbHash) require.Nil(t, err) @@ -603,6 +595,9 @@ func TestStorageUploadUpdate(t *testing.T) { h = sha3.New256() _, err = io.Copy(h, f) require.Nil(t, err) + pathWriter = fs.pathWriter(fid.Path) + _, err = io.Copy(h, pathWriter) + require.Nil(t, err) require.Equal(t, hex.EncodeToString(h.Sum(nil)), prevThumbHash) f.Close() } @@ -801,6 +796,40 @@ func TestGetMerkleTree(t *testing.T) { } } +func TestValidationRoot(t *testing.T) { + + thumbnailBytes, _ := base64.StdEncoding.DecodeString(`iVBORw0KGgoAAAANSUhEUgAAANgAAADpCAMAAABx2AnXAAAAwFBMVEX///8REiQAAADa2ttlZWWlpaU5OTnIyMiIiIhzc3ODg4OVlZXExMT6+vr39/fOzs7v7+9dXV0rKyvf399GRkbn5+dBQUEREREAABp5eXmxsbFsbGxaWlqfn59gYGC4uLgAABWrq6sAAByXl5dOTk4LCwscHBwvLy88PDwkJCR5eYGUlJpBQUxtbnYAAA8ZGyojJTNiY2sAAB82N0OFhYxSU10uLjxKSlQeHy1+f4ebnaRNUFmLjZNdXWWqq7JoaXKY6lzbAAAMKUlEQVR4nO2dC1u6PhvHETARORlhchA8ZYVa+tM0+2u9/3f17N5AUdG0ELBnn666pgzal+3e4d4GDEOhUCgUCoVCoVAoFAqFQqFQKBQKhUKhUCiUP4pqPrNst2NknY6E0Rw2oJh1Us7FsIotST508IFdY6aarN+i1oJUa3FHlWc2QiftxP0CYZNsNeZwBQ48Whwn4ijXY2eVaIbo+8fh6y4uphIEhbTT91NULOjRde5xoPYU4AQVRSmSTXAPnrNL6nncQcItFNBsdps7BY63IMOCuBx8rcRdRZMqQkM9VP1kgQ5pbZFwd0eZCF8WUcANIhvwbUwNIxPzY5+tlFJ9AthugnBrR9gzZI6FAjeRyA/719A37YGTm0wDMU4QBg01iWCFmYNzqYGPy7VIsdygRW+Gs3c4I0DAUxCOljplXeqwEQqo+ijh5s4L4nZrIaSd4wUcMTedEzViNm5oV0yQDdo6xpoaOeyw2zhQatUeCt3HVi7pI4N9kGbKimRIRBjOyJCesfcV8EhMC9eaUvoiYsH9jhtP54R1fQFEhBHFmKegQYutPxmSkblpwXvRFIYZtiWM0UQcqbauzcGcKkE140bEdFC4nGbij6Hfb3Rt7vaWMGJoN5tzQFgpCAuRHBMj4ewx1gUrUqPtCJP2hYW2BPYW9rPgpNbFE3w6Eo+qkOdKtE9xujB9k9VlCMb0o7Nkt8dwujCmClHdkuHhhoy/dEp/yRnC9K0KMnawmiPOEMZ4EV1xQ9VccY4wphR6D2pcikn8GWcJY5SW+/xwY+el03GM84QhZDk3I5ajnC3sWqDCro2/LUxhDE5VOc7ATri/IQxcAw/8DWmeHm6628K6eW+KFZQh8UjsEfBA56brOLxdNkVBqHQaiGKxZVmeJ0kllcvWP2DtDoQT5C670YtROymF988P30eK4yaj6Qv9+6SxrkcSp/8sbzPpOMq3+H8/3+xzR7Ko24iOQLjAsy9gq4RKpeJZrWKjUxEE0TTLts3zrus4Trd7V7shneJeFpaGJ4+eVEXeI3BK7bku9Cf8Pa4Moz6PfWRZUe9ir5ECOE9ij2DnYOzMpYmPQOk8oR3D4+r0+8XRWa8dcBltxB6qhLfjBGG4hU+/EYe5iLvYIzjxh5ye2FvT+q4oEpwD+X5ZDno2tcNlFIBao2cJ4D8VveO1XtTfmB6VQ8KEw2UU2J6hYMUj2vIlTOl9k5zd+VznoLR8CcNdxGMeNG6vGT5kj/kSBjX6cZcnilErFy3BdMIuWS3+RuRL2CNLlhAcQV/7sI0i6b7cxirLlTAZ0nmG811uYGWPcX2nXAmDnvHzWU5q4/ZQ+5AbYZxXEXl2Pct8Kgo2NVsUi+r2HcmHMKXyGNZyh1vneLT16riHatRdkAthnUj1Hd/TOkJ0ZBdx3udAmHYTbZfOn+DaWj+3dglkL0wPptd75UrF7jk/mOCqOGJFDAfZYYOdubBgZaz4+ylWj+R8hXzKXBhOzU0yM8ekUJJRWNbCcL2R2KI1PLlJfB0ZC8Pjr6fkhvDWujBmLAwXniQ9gHyYZdkKk8HCEl1Mj9c3wsqlbIXpSWcYGYrCpbMV1jq/c/gdUH/0mKyFCUmXxKAQMFkLMzcNalJoMMmkZS0MHIXxztEfo/WI2WYrTGQTXxIaLs7P3sYSXhLK5cLGcBWW7NQBuEFgwXu2wnC5SXaa/C4o3Rl3qWAUda4z4ChqeKsyFuaFPaCk6IVNftbDFuw+S262uLy+UVkLw976+6SU4UlP4g7KWhhD9n4lstdGJ74B4jXJXBiZLWYfG/qvJvllQwqmmIJKNnthcri16DZmbcTJrB2ucTsoshG2tWH4tzwa0YtmLYzhqsnI6kU61LkQhqQJt7+WxVtRK82JMARX+hW7nsn8CEsYKixR/qywFPYcZiMMtuldeC829EMS9hOdAO76XnSdpAzOqiTHQ6eBN6Zf9DkxuDeTwS45PG6Kf5ZMEih4zOB+HzFxgicfdPmL0CWzpJms4z66YyAZ0rewdJRlpAuVRvOSsuxMH4ckWcUjwJKbu9b+9y3w2d0fO9M6+PSuPIDng2LXYa99h9eGoSMM6Do8xt95WBjm4Fh6nrNmh1LEUg44r6xIlPw8DeIbtlb9Huh1ydGHgOTmySTfIJ6SG1vrwtJM3S+AhRoP98BD97ABOSQK3vuX9+cmBICwhqwAx6LhCIpxf13CTnZ4a1RY9lBhwLUJE3Ruza4j1OAilK5M2Bbb+yB2tyNdj7D9qZfoXu393UhX00Brexu6oyNGY19Xnp6wdRSDv91iu1/V2j54W8tsoPwDSL8jYLdbtXXweO+EQqFQKBQKhUKhUCgUCoVCoVAoFMoB5PC5xmtXu3zhR8KmNGdWqlYdoLt+rpvUvdCyO3LHODedyaVSVTUw66kTqXohYVIXMkvn03l5XKm6O5N8OWHVNGdut4RpXtGTS0SY2ipKgd2prVZkCaIsFS0ujG7pJKDAmYxabAU3hUNn4zLgkQiWjH5dFT54GnxGcYsqs32ZiwlTed60+YZrwCLyatl0bTimmK5pukJYVA2IVIVtbpK7Cdl22RUrbpl3seZO1TZ5OFvh8YY41eGYMm/zVY7RwJol1+TLtotXx5HLJP46uRIvIkz8VklXNOBtSDz62+HR7TRMHskRTQNMPrAMuQwfJVthdBdemWRVPTingnIClBhl2IvQciU4G0VSbJxiFSlSUI4Z8N5eD/6rAOe6KKhX8WWcpOd10b/odDoVWAfr8TjzIMc0HlddHEqgQR6y2go2T0ASGfzCpAZPHjJlgvWsM6fBo4M4GxkDaY4IC2yMCCMZa4roBFsjl0l4QWqkKHZI2lXHYDiiRrZbqHyaZYRtE4OzqmF0kUyteyhhuL6R+WIgTHeI9ZQbO8KMjTA9vCkmWa3puQnPWUeENcoy+cYIkwbJUnkLv/4tsHSrGt5ZgQizQmFKRBjZGIzOPphja2GiEFz3csJK5OmOUCg0Gz9SuoTSqmyXfq4art5u8bgGhOK0K8zFm6hUR2JkExcDzz2YY+Fl+KSFuZIerrk27ZJiNHDKi25RU6Qy3O9W1VMYbv2kZoGXFM1CajTe5BSjAndjVxjPdzSlxIPZeG4DXcjmObA5gdOIMGkjTOPL6DJCOXFhkS6VVkHh4P1MDd5xylwZ0mqhYFUIG1e54joO7j0YphNEx70wGVfZxSpUdJ6AThHxKQ0U3W44uAXjnQaq7iHHSLdNgK2FHFymmLiNyeFqNXxdY/OWDhSUNR4XQ41To50RQw0ftqoH0UkvUMcmpIOwEjqkb6KjHGfIhVB0eHBB0NHWDHI2unzDTmeZvoAr7MZPHoJJhJ2Mire6GG5KL3yVqqblidWftZphrXgSillteEXXTGuFElcp28IPN6kYzjknKpZom60UV1794nVo56byinbBUCgUCoVCoVAoFAqFQqFQKBQK5fJwfxQmZuf/n4Ap/FGosGvjqLB6e+tT8HsdBMIm6Hf0ugljmqu35mz96XVeL4xWk8KVQIS1v8b15rLZbBbqTXb5Wm826yjQ+vz8HH6wLyxbqLPsTGXZyXSQcXpPJsix92XzfeH3p+yi7y/6s37fn3/8x/3HskNtteTU2YDj5tKAmw1SzbF6XMnfMY92uw3fwd961FQCYc1l4Ws4bA6HY5ad/lsW2KH/9jJQ9cWwP1LZ8ac0YUcGF/uPLsdsuJq811/fB81RuzBY/jeoj+qF1ylK/gz9FF7fm+PV9G25mE9Xk+V4OZuu2M+2v6hHhdVRlFV//OUP6s3pv4+X5td03n5h29yiM/fYiVd6eRkZ6qh9JBnJ0576w8/hdP658v3PwXLyOfS/lnNvyPqr4XDR7y/GPuu/fS5Zf7zq+NNFcfhWZP2vdlRYof3pvy/rs1G/8L4aD1eF/uqt/TFcllDx44aS3/f8QWnOvaQqrL5AyubLwYc/XnZmX8uP6XjxMfmcjpbzxbj/tZx8vPn+YPkxHE6m1r/+23LpS7NVv7ktbPjeni39+mjpv4zZr+n7bFZ/qyzqzdX8X3/18jLsz4bsMOWqAxW2QWE2eS0MUNEbtGdtVCgno9mkOa8P6u+jwmA0exvMXtGfl9Fo0pyNXkbtMInrdgwyEGyoWQeLxKrbzTr+rgmGiSrMPLZi9fWfHf4/ex7XDBV2bfwPF18HmekEj6sAAAAASUVORK5CYII=`) + size := int64(3374) + + fs, cleanUp := setupStorage(t) + defer cleanUp() + fPath := filepath.Join(fs.mp, randString(10)+".txt") + cH := GetNewCommitHasher(size) + _, err := cH.Write(thumbnailBytes) + require.Nil(t, err) + + err = cH.Finalize() + require.Nil(t, err) + + f, err := os.Create(fPath) + require.Nil(t, err) + defer f.Close() + + validationMerkleRoot, err := cH.vt.CalculateRootAndStoreNodes(f) + require.Nil(t, err) + + bufReader := bytes.NewReader(thumbnailBytes) + pathWriter := fs.pathWriter(fPath) + h := sha3.New256() + _, err = io.Copy(h, pathWriter) + require.Nil(t, err) + _, err = io.Copy(h, bufReader) + require.Nil(t, err) + + hash := hex.EncodeToString(h.Sum(nil)) + require.NotEqual(t, hash, hex.EncodeToString(validationMerkleRoot)) +} + func setupStorage(t *testing.T) (*FileStore, func()) { wd, err := os.Getwd() diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation.go b/code/go/0chain.net/blobbercore/filestore/tree_validation.go index 2aa9f9866..6569d4758 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation.go @@ -6,6 +6,7 @@ package filestore import ( "encoding/hex" "errors" + "fmt" "io" "math" @@ -138,7 +139,7 @@ func (fp fixedMerkleTreeProof) GetMerkleProof(r io.ReaderAt) (proof [][]byte, er b := make([]byte, FMTSize) n, err := r.ReadAt(b, io.SeekStart) if n != FMTSize { - return nil, errors.New("incomplete read") + return nil, fmt.Errorf("invalid fixed merkle tree size: %d", n) } if err != nil { return nil, err @@ -356,9 +357,9 @@ func (v *validationTreeProof) getFileOffsetsAndNodeIndexes(startInd, endInd int) } // getNodeIndexes returns two slices. -// 1. NodeOffsets will return offset index of node in each level. Each level starts with index zero. -// 2. leftRightIndexes will return whether the node should be appended to the left or right -// with other hash +// 1. NodeOffsets will return offset index of node in each level. Each level starts with index zero. +// 2. leftRightIndexes will return whether the node should be appended to the left or right +// with other hash func (v *validationTreeProof) getNodeIndexes(startInd, endInd int) ([][]int, [][]int) { indexes := make([][]int, 0) diff --git a/code/go/0chain.net/blobbercore/handler/download_quota.go b/code/go/0chain.net/blobbercore/handler/download_quota.go index 6ea2d202f..8d731cc5f 100644 --- a/code/go/0chain.net/blobbercore/handler/download_quota.go +++ b/code/go/0chain.net/blobbercore/handler/download_quota.go @@ -56,9 +56,9 @@ func (qm *QuotaManager) createOrUpdateQuota(numBlocks int64, key string) { func (qm *QuotaManager) consumeQuota(key string, numBlocks int64) error { qm.mux.Lock() - dq, ok := qm.m[key] - qm.mux.Unlock() + defer qm.mux.Unlock() + dq, ok := qm.m[key] if !ok { return common.NewError("consume_quota", "no download quota") } diff --git a/code/go/0chain.net/blobbercore/handler/download_request_header.go b/code/go/0chain.net/blobbercore/handler/download_request_header.go index b18d7a29e..bda1fda40 100644 --- a/code/go/0chain.net/blobbercore/handler/download_request_header.go +++ b/code/go/0chain.net/blobbercore/handler/download_request_header.go @@ -24,7 +24,6 @@ type DownloadRequestHeader struct { AuthToken string VerifyDownload bool DownloadMode string - SubmitRM bool ConnectionID string } @@ -96,7 +95,6 @@ func (dr *DownloadRequestHeader) Parse(isRedeem bool) error { if err != nil { return errors.Throw(common.ErrInvalidParameter, "X-Read-Marker") } - dr.SubmitRM = true } else if isRedeem { return errors.Throw(common.ErrInvalidParameter, "X-Read-Marker") } diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index cfb496762..fd8b7c1d8 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -53,6 +53,7 @@ func (cmd *UpdateFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("invalid_parameters", "Invalid parameters. Error parsing the meta data for upload."+err.Error()) } + logging.Logger.Info("UpdateFileCommand", zap.Any("allocation_id", allocationObj.ID), zap.Any("validation_rooot", cmd.fileChanger.ValidationRoot), zap.Any("thumb_hash", cmd.fileChanger.ThumbnailHash)) if cmd.fileChanger.ChunkSize <= 0 { cmd.fileChanger.ChunkSize = fileref.CHUNK_SIZE diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index ddfb05f8e..49d3bfb46 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -58,6 +58,10 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request "Invalid parameters. Error parsing the meta data for upload."+err.Error()) } + if fileChanger.Path == "/" { + return common.NewError("invalid_path", "Invalid path. Cannot upload to root directory") + } + if !filepath.IsAbs(fileChanger.Path) { return common.NewError("invalid_path", fmt.Sprintf("%v is not absolute path", fileChanger.Path)) } diff --git a/code/go/0chain.net/blobbercore/handler/handler_download_test.go b/code/go/0chain.net/blobbercore/handler/handler_download_test.go index 4d8e92971..2547c75e7 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_download_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_download_test.go @@ -13,13 +13,11 @@ import ( "testing" "time" - "github.com/0chain/blobber/code/go/0chain.net/core/node" - "github.com/0chain/gosdk/core/zcncrypto" "github.com/0chain/gosdk/zboxcore/client" zencryption "github.com/0chain/gosdk/zboxcore/encryption" "github.com/0chain/gosdk/zboxcore/fileref" - "github.com/0chain/gosdk/zboxcore/marker" + "github.com/0chain/gosdk/zboxcore/zboxutil" "github.com/DATA-DOG/go-sqlmock" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" @@ -216,21 +214,7 @@ func TestHandlers_Download(t *testing.T) { } remotePath := "/file.txt" - - rm := &marker.ReadMarker{} - rm.ClientID = ownerClient.ClientID - rm.ClientPublicKey = ownerClient.ClientKey - rm.BlobberID = node.Self.ID - rm.AllocationID = alloc.ID - rm.OwnerID = ownerClient.ClientID - rm.ReadCounter = 1 - rm.SessionRC = 1 - rm.Signature, err = signHash(ownerClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - rmData, err := json.Marshal(rm) - require.NoError(t, err) + connectionID := zboxutil.NewConnectionId() r, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { require.NoError(t, err) @@ -242,11 +226,12 @@ func TestHandlers_Download(t *testing.T) { t.Fatal(err) } + r.Header.Set("X-Path", remotePath) r.Header.Set("X-Path-Hash", fileref.GetReferenceLookup(alloc.Tx, remotePath)) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Submit-RM", fmt.Sprint(true)) - r.Header.Set("X-Read-Marker", string(rmData)) + r.Header.Set("X-Connection-ID", connectionID) + r.Header.Set("X-Mode", DownloadContentFull) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, alloc.OwnerID) r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) @@ -299,37 +284,23 @@ func TestHandlers_Download(t *testing.T) { } remotePath := "/file.txt" - - rm := &marker.ReadMarker{} - rm.ClientID = ownerClient.ClientID - rm.ClientPublicKey = ownerClient.ClientKey - rm.BlobberID = node.Self.ID - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.SessionRC = 1 - rm.OwnerID = ownerClient.ClientID - rm.Signature, err = signHash(ownerClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - rmData, err := json.Marshal(rm) - require.NoError(t, err) + connectionID := zboxutil.NewConnectionId() r, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { t.Fatal(err) } - hash := encryption.Hash(alloc.Tx) sign, err := sch.Sign(hash) if err != nil { t.Fatal(err) } + r.Header.Set("X-Path", remotePath) r.Header.Set("X-Path-Hash", fileref.GetReferenceLookup(alloc.Tx, remotePath)) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Submit-RM", fmt.Sprint(true)) - r.Header.Set("X-Read-Marker", string(rmData)) + r.Header.Set("X-Connection-ID", connectionID) + r.Header.Set("X-Mode", DownloadContentFull) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, alloc.OwnerID) r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) @@ -369,126 +340,113 @@ func TestHandlers_Download(t *testing.T) { sqlmock.NewRows([]string{"path", "type", "lookup_hash", "validation_root"}). AddRow("/file.txt", "f", filePathHash, "abcd"), ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(ownerClient.ClientID, alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"client_id"}). - AddRow(ownerClient.ClientID), - ) - - aa := sqlmock.AnyArg() - - mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(aa, aa, aa, aa, aa, aa). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectCommit() }, wantCode: http.StatusOK, wantBody: "bW9jaw==", //base64encoded for mock string }, - { - name: "DownloadFile_file_return_stale_readmarker", - args: args{ - w: httptest.NewRecorder(), - r: func() *http.Request { - handlerName := handlers["/v1/file/download/{allocation}"] - url, err := router.Get(handlerName).URL("allocation", alloc.Tx) - if err != nil { - t.Fatal() - } - - remotePath := "/file.txt" - - rm := &marker.ReadMarker{} - rm.ClientID = ownerClient.ClientID - rm.ClientPublicKey = ownerClient.ClientKey - rm.BlobberID = node.Self.ID - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.SessionRC = 1 - rm.OwnerID = ownerClient.ClientID - rm.Signature, err = signHash(ownerClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - rmData, err := json.Marshal(rm) - require.NoError(t, err) - r, err := http.NewRequest(http.MethodGet, url.String(), nil) - if err != nil { - t.Fatal(err) - } - - hash := encryption.Hash(alloc.Tx) - sign, err := sch.Sign(hash) - if err != nil { - t.Fatal(err) - } - - r.Header.Set("X-Path-Hash", fileref.GetReferenceLookup(alloc.Tx, remotePath)) - r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Submit-RM", fmt.Sprint(true)) - r.Header.Set("X-Read-Marker", string(rmData)) - r.Header.Set(common.ClientSignatureHeader, sign) - r.Header.Set(common.ClientHeader, alloc.OwnerID) - r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) - r.Header.Set(common.AllocationIdHeader, alloc.ID) - - return r - }(), - }, - alloc: alloc, - setupDbMock: func(mock sqlmock.Sqlmock) { - mock.ExpectBegin() - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). - WithArgs(alloc.Tx). - WillReturnRows( - sqlmock.NewRows( - []string{ - "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", - }, - ). - AddRow( - alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), - ), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). - WithArgs(alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"id", "allocation_id"}). - AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), - ) - - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). - WithArgs(alloc.ID, filePathHash). - WillReturnRows( - sqlmock.NewRows([]string{"path", "type", "lookup_hash", "validation_root"}). - AddRow("/file.txt", "f", filePathHash, "abcd"), - ) - - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(ownerClient.ClientID, alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"client_id", "counter"}). - AddRow(ownerClient.ClientID, 23), - ) - - aa := sqlmock.AnyArg() - - mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(aa, aa, aa, aa). - WillReturnResult(sqlmock.NewResult(0, 0)) - - mock.ExpectCommit() - }, - wantCode: http.StatusBadRequest, - wantBody: "{\"code\":\"stale_read_marker\",\"error\":\"stale_read_marker: \"}\n\n", - }, + //TODO: redeeming is moved to a separate route + // { + // name: "DownloadFile_file_return_stale_readmarker", + // args: args{ + // w: httptest.NewRecorder(), + // r: func() *http.Request { + // handlerName := handlers["/v1/file/download/{allocation}"] + // url, err := router.Get(handlerName).URL("allocation", alloc.Tx) + // if err != nil { + // t.Fatal() + // } + + // remotePath := "/file.txt" + + // rm := &marker.ReadMarker{} + // rm.ClientID = ownerClient.ClientID + // rm.ClientPublicKey = ownerClient.ClientKey + // rm.BlobberID = node.Self.ID + // rm.AllocationID = alloc.ID + // rm.ReadCounter = 1 + // rm.SessionRC = 1 + // rm.OwnerID = ownerClient.ClientID + // rm.Signature, err = signHash(ownerClient, rm.GetHash()) + // if err != nil { + // t.Fatal(err) + // } + // rmData, err := json.Marshal(rm) + // require.NoError(t, err) + // r, err := http.NewRequest(http.MethodGet, url.String(), nil) + // if err != nil { + // t.Fatal(err) + // } + + // hash := encryption.Hash(alloc.Tx) + // sign, err := sch.Sign(hash) + // if err != nil { + // t.Fatal(err) + // } + + // r.Header.Set("X-Path-Hash", fileref.GetReferenceLookup(alloc.Tx, remotePath)) + // r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) + // r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) + // r.Header.Set("X-Submit-RM", fmt.Sprint(true)) + // r.Header.Set("X-Read-Marker", string(rmData)) + // r.Header.Set(common.ClientSignatureHeader, sign) + // r.Header.Set(common.ClientHeader, alloc.OwnerID) + // r.Header.Set(common.ClientKeyHeader, alloc.OwnerPublicKey) + // r.Header.Set(common.AllocationIdHeader, alloc.ID) + + // return r + // }(), + // }, + // alloc: alloc, + // setupDbMock: func(mock sqlmock.Sqlmock) { + // mock.ExpectBegin() + + // mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "allocations" WHERE`)). + // WithArgs(alloc.Tx). + // WillReturnRows( + // sqlmock.NewRows( + // []string{ + // "id", "tx", "expiration_date", "owner_public_key", "owner_id", "blobber_size", + // }, + // ). + // AddRow( + // alloc.ID, alloc.Tx, alloc.Expiration, alloc.OwnerPublicKey, alloc.OwnerID, int64(1<<30), + // ), + // ) + + // mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "terms" WHERE`)). + // WithArgs(alloc.ID). + // WillReturnRows( + // sqlmock.NewRows([]string{"id", "allocation_id"}). + // AddRow(alloc.Terms[0].ID, alloc.Terms[0].AllocationID), + // ) + + // filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") + // mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "reference_objects" WHERE`)). + // WithArgs(alloc.ID, filePathHash). + // WillReturnRows( + // sqlmock.NewRows([]string{"path", "type", "lookup_hash", "validation_root"}). + // AddRow("/file.txt", "f", filePathHash, "abcd"), + // ) + + // mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). + // WithArgs(ownerClient.ClientID, alloc.ID). + // WillReturnRows( + // sqlmock.NewRows([]string{"client_id", "counter"}). + // AddRow(ownerClient.ClientID, 23), + // ) + + // aa := sqlmock.AnyArg() + + // mock.ExpectExec(`UPDATE "read_markers"`). + // WithArgs(aa, aa, aa, aa). + // WillReturnResult(sqlmock.NewResult(0, 0)) + + // mock.ExpectCommit() + // }, + // wantCode: http.StatusBadRequest, + // wantBody: "{\"code\":\"stale_read_marker\",\"error\":\"stale_read_marker: \"}\n\n", + // }, { name: "DownloadFile_Encrypted_Permission_Denied_Unshared_File", args: args{ @@ -501,43 +459,28 @@ func TestHandlers_Download(t *testing.T) { } remotePath := "/file.txt" - + connectionID := zboxutil.NewConnectionId() pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, ownerClient.Keys[0].PublicKey) if err != nil { t.Fatal(err) } - - rm := &marker.ReadMarker{} - rm.ClientID = guestClient.ClientID - rm.ClientPublicKey = guestClient.ClientKey - rm.BlobberID = node.Self.ID - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.SessionRC = 1 - rm.OwnerID = ownerClient.ClientID - rm.Signature, err = signHash(guestClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - rmData, err := json.Marshal(rm) - require.NoError(t, err) r, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { t.Fatal(err) } - hash := encryption.Hash(alloc.Tx) sign, err := sch.Sign(hash) if err != nil { t.Fatal(err) } - r.Header.Set("X-Read-Marker", string(rmData)) r.Header.Set("X-Path-Hash", pathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Submit-RM", fmt.Sprint(true)) + r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Connection-ID", connectionID) + r.Header.Set("X-Mode", DownloadContentFull) r.Header.Set("X-Auth-Token", authTicket) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, guestClient.ClientID) @@ -598,42 +541,27 @@ func TestHandlers_Download(t *testing.T) { } remotePath := "/file.txt" - + connectionID := zboxutil.NewConnectionId() pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") if err != nil { t.Fatal(err) } - rm := &marker.ReadMarker{} - rm.ClientID = guestClient.ClientID - rm.ClientPublicKey = guestClient.ClientKey - rm.BlobberID = node.Self.ID - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.SessionRC = 1 - rm.OwnerID = ownerClient.ClientID - rm.Signature, err = signHash(guestClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - rmData, err := json.Marshal(rm) - require.NoError(t, err) r, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { t.Fatal(err) } - hash := encryption.Hash(alloc.Tx) sign, err := sch.Sign(hash) if err != nil { t.Fatal(err) } - - r.Header.Set("X-Read-Marker", string(rmData)) r.Header.Set("X-Path-Hash", pathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Submit-RM", fmt.Sprint(true)) + r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Connection-ID", connectionID) + r.Header.Set("X-Mode", DownloadContentFull) r.Header.Set("X-Auth-Token", authTicket) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, guestClient.ClientID) @@ -707,19 +635,6 @@ func TestHandlers_Download(t *testing.T) { AddRow(reEncryptionKey, guestPublicEncryptedKey), ) - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(guestClient.ClientID, alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"client_id"}). - AddRow(guestClient.ClientID), - ) - - aa := sqlmock.AnyArg() - - mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(aa, aa, aa, aa, aa, aa). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectCommit() }, wantCode: http.StatusOK, @@ -737,29 +652,13 @@ func TestHandlers_Download(t *testing.T) { } remotePath := "/" + connectionID := zboxutil.NewConnectionId() pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) - filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/file.txt") authTicket, err := GetAuthTicketForEncryptedFile(ownerClient, alloc.ID, remotePath, pathHash, guestClient.ClientID, "") if err != nil { t.Fatal(err) } - rm := &marker.ReadMarker{} - rm.ClientID = guestClient.ClientID - rm.ClientPublicKey = guestClient.ClientKey - rm.BlobberID = node.Self.ID - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.SessionRC = 1 - rm.OwnerID = ownerClient.ClientID - rm.Signature, err = signHash(guestClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - - rmData, err := json.Marshal(rm) - require.NoError(t, err) - r, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { t.Fatal(err) @@ -773,10 +672,11 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", filePathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Auth-Token", authTicket) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Submit-RM", fmt.Sprint(true)) - r.Header.Set("X-Read-Marker", string(rmData)) + r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Connection-ID", connectionID) + r.Header.Set("X-Mode", DownloadContentFull) + r.Header.Set("X-Auth-Token", authTicket) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, guestClient.ClientID) r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) @@ -853,19 +753,6 @@ func TestHandlers_Download(t *testing.T) { AddRow(reEncryptionKey, gpbk), ) - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(guestClient.ClientID, alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"client_id"}). - AddRow(guestClient.ClientID), - ) - - aa := sqlmock.AnyArg() - - mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(aa, aa, aa, aa, aa, aa). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectCommit() }, wantCode: http.StatusOK, @@ -883,6 +770,7 @@ func TestHandlers_Download(t *testing.T) { } remotePath := "/folder1" + connectionID := zboxutil.NewConnectionId() pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder1/subfolder1/file.txt") @@ -890,21 +778,6 @@ func TestHandlers_Download(t *testing.T) { if err != nil { t.Fatal(err) } - rm := &marker.ReadMarker{} - rm.ClientID = guestClient.ClientID - rm.ClientPublicKey = guestClient.ClientKey - rm.BlobberID = node.Self.ID - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.SessionRC = 1 - rm.OwnerID = alloc.OwnerID - rm.Signature, err = signHash(guestClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - - rmData, err := json.Marshal(rm) - require.NoError(t, err) r, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { @@ -919,10 +792,11 @@ func TestHandlers_Download(t *testing.T) { r.Header.Set("X-Path-Hash", filePathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Auth-Token", authTicket) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Submit-RM", fmt.Sprint(true)) - r.Header.Set("X-Read-Marker", string(rmData)) + r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Connection-ID", connectionID) + r.Header.Set("X-Mode", DownloadContentFull) + r.Header.Set("X-Auth-Token", authTicket) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, guestClient.ClientID) r.Header.Set(common.ClientKeyHeader, guestClient.ClientKey) @@ -999,19 +873,6 @@ func TestHandlers_Download(t *testing.T) { AddRow(reEncryptionKey, gpbk), ) - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "read_markers" WHERE`)). - WithArgs(guestClient.ClientID, alloc.ID). - WillReturnRows( - sqlmock.NewRows([]string{"client_id"}). - AddRow(guestClient.ClientID), - ) - - aa := sqlmock.AnyArg() - - mock.ExpectExec(`UPDATE "read_markers"`). - WithArgs(aa, aa, aa, aa, aa, aa). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectCommit() }, wantCode: http.StatusOK, @@ -1029,6 +890,7 @@ func TestHandlers_Download(t *testing.T) { } remotePath := "/folder1" + connectionID := zboxutil.NewConnectionId() pathHash := fileref.GetReferenceLookup(alloc.Tx, remotePath) filePathHash := fileref.GetReferenceLookup(alloc.Tx, "/folder2/subfolder1/file.txt") @@ -1036,22 +898,6 @@ func TestHandlers_Download(t *testing.T) { if err != nil { t.Fatal(err) } - rm := &marker.ReadMarker{} - rm.ClientID = guestClient.ClientID - rm.ClientPublicKey = guestClient.ClientKey - rm.BlobberID = node.Self.ID - rm.AllocationID = alloc.ID - rm.ReadCounter = 1 - rm.SessionRC = 1 - rm.OwnerID = alloc.OwnerID - rm.Signature, err = signHash(guestClient, rm.GetHash()) - if err != nil { - t.Fatal(err) - } - - rmData, err := json.Marshal(rm) - require.NoError(t, err) - r, err := http.NewRequest(http.MethodGet, url.String(), nil) if err != nil { t.Fatal(err) @@ -1063,11 +909,12 @@ func TestHandlers_Download(t *testing.T) { t.Fatal(err) } - r.Header.Set("X-Read-Marker", string(rmData)) r.Header.Set("X-Path-Hash", filePathHash) r.Header.Set("X-Block-Num", fmt.Sprintf("%d", 1)) r.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", 1)) - r.Header.Set("X-Submit-RM", fmt.Sprint(true)) + r.Header.Set("X-Verify-Download", fmt.Sprint(false)) + r.Header.Set("X-Connection-ID", connectionID) + r.Header.Set("X-Mode", DownloadContentFull) r.Header.Set("X-Auth-Token", authTicket) r.Header.Set(common.ClientSignatureHeader, sign) r.Header.Set(common.ClientHeader, guestClient.ClientID) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 579c1f162..e2f8d4c08 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -168,6 +168,8 @@ func (fsh *StorageHandler) RedeemReadMarker(ctx context.Context, r *http.Request allocationTx = ctx.Value(constants.ContextKeyAllocation).(string) allocationID = ctx.Value(constants.ContextKeyAllocationID).(string) alloc *allocation.Allocation + blobberID = node.Self.ID + quotaManager = getQuotaManager() ) if clientID == "" { @@ -184,9 +186,17 @@ func (fsh *StorageHandler) RedeemReadMarker(ctx context.Context, r *http.Request return nil, err } - key := clientID + ":" + alloc.ID - quotaManager := getQuotaManager() + isReadFree := alloc.IsReadFree(blobberID) + if isReadFree { + Logger.Info("free_read: readmarker not saved", + zap.String("clientID", clientID), + zap.String("allocationID", allocationID)) + return &blobberhttp.DownloadResponse{ + Success: true, + }, nil + } + key := clientID + ":" + alloc.ID lock, isNewLock := readmarker.ReadmarkerMapLock.GetLock(key) if !isNewLock { return nil, common.NewErrorf("lock_exists", fmt.Sprintf("lock exists for key: %v", key)) @@ -263,6 +273,8 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i allocationTx = ctx.Value(constants.ContextKeyAllocation).(string) allocationID = ctx.Value(constants.ContextKeyAllocationID).(string) alloc *allocation.Allocation + blobberID = node.Self.ID + quotaManager = getQuotaManager() ) if clientID == "" { @@ -288,9 +300,6 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i return nil, common.NewErrorf("download_file", "path is not a file: %v", err) } - key := clientID + ":" + alloc.ID - quotaManager := getQuotaManager() - isOwner := clientID == alloc.OwnerID var authToken *readmarker.AuthTicket @@ -322,85 +331,19 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i } - if dr.SubmitRM { - lock, isNewLock := readmarker.ReadmarkerMapLock.GetLock(key) - if !isNewLock { - return nil, common.NewErrorf("lock_exists", fmt.Sprintf("lock exists for key: %v", key)) - } - - lock.Lock() - defer lock.Unlock() - - // create read marker - var ( - rme *readmarker.ReadMarkerEntity - latestRM *readmarker.ReadMarker - latestRedeemedRC int64 - pendNumBlocks int64 - ) - - rme, err = readmarker.GetLatestReadMarkerEntity(ctx, clientID, alloc.ID) - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - return nil, common.NewErrorf("download_file", "couldn't get read marker from DB: %v", err) - } - - if rme != nil { - latestRM = rme.LatestRM - latestRedeemedRC = rme.LatestRedeemedRC - if pendNumBlocks, err = rme.PendNumBlocks(); err != nil { - return nil, common.NewErrorf("download_file", "couldn't get number of blocks pending redeeming: %v", err) - } - } + isReadFree := alloc.IsReadFree(blobberID) + var dq *DownloadQuota - // check out read pool tokens if read_price > 0 - err = readPreRedeem(ctx, alloc, dr.ReadMarker.SessionRC, pendNumBlocks, clientID) - if err != nil { - return nil, common.NewErrorf("not_enough_tokens", "pre-redeeming read marker: %v", err) + if !isReadFree { + dq = quotaManager.getDownloadQuota(dr.ConnectionID) + if dq == nil { + return nil, common.NewError("download_file", fmt.Sprintf("no download quota for %v", dr.ConnectionID)) } - - if latestRM != nil && latestRM.ReadCounter+(dr.ReadMarker.SessionRC) > dr.ReadMarker.ReadCounter { - latestRM.BlobberID = node.Self.ID - return &blobberhttp.DownloadResponse{ - Success: false, - LatestRM: latestRM, - Path: fileref.Path, - AllocationID: fileref.AllocationID, - }, common.NewError("stale_read_marker", "") - } - - if dr.ReadMarker.ClientID != clientID { - return nil, common.NewError("invalid_client", "header clientID and readmarker clientID are different") - } - - rmObj := new(readmarker.ReadMarkerEntity) - rmObj.LatestRM = &dr.ReadMarker - - if err = rmObj.VerifyMarker(ctx, alloc); err != nil { - return nil, common.NewErrorf("download_file", "invalid read marker, "+"failed to verify the read marker: %v", err) - } - - err = readmarker.SaveLatestReadMarker(ctx, &dr.ReadMarker, latestRedeemedRC, latestRM == nil) - if err != nil { - Logger.Error(err.Error()) - return nil, common.NewErrorf("download_file", "couldn't save latest read marker") - } - - quotaManager.createOrUpdateQuota(dr.ReadMarker.SessionRC, dr.ConnectionID) - - if dr.NumBlocks == 0 { - return nil, nil + if dq.Quota < dr.NumBlocks { + return nil, common.NewError("download_file", fmt.Sprintf("insufficient quota: available %v, requested %v", dq.Quota, dr.NumBlocks)) } } - dq := quotaManager.getDownloadQuota(dr.ConnectionID) - if dq == nil { - return nil, common.NewError("download_file", fmt.Sprintf("no download quota for %v", dr.ConnectionID)) - } - - if dq.Quota < dr.NumBlocks { - return nil, common.NewError("download_file", fmt.Sprintf("insufficient quota: available %v, requested %v", dq.Quota, dr.NumBlocks)) - } - var ( downloadMode = dr.DownloadMode fileDownloadResponse *filestore.FileDownloadResponse @@ -468,9 +411,11 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i return nil, err } - err = quotaManager.consumeQuota(dr.ConnectionID, dr.NumBlocks) - if err != nil { - return nil, common.NewError("download_file", err.Error()) + if !isReadFree { + err = quotaManager.consumeQuota(dr.ConnectionID, dr.NumBlocks) + if err != nil { + return nil, common.NewError("download_file", err.Error()) + } } fileDownloadResponse.Data = chunkData @@ -720,13 +665,16 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b if err = db.Model(allocationObj).Updates(allocationUpdates).Error; err != nil { return nil, common.NewError("allocation_write_error", "Error persisting the allocation object") } - err = connectionObj.CommitToFileStore(ctx) if err != nil { if !errors.Is(common.ErrFileWasDeleted, err) { return nil, common.NewError("file_store_error", "Error committing to file store. "+err.Error()) } } + err = writemarkerEntity.SendToChan(ctx) + if err != nil { + return nil, common.NewError("write_marker_error", "Error redeeming the write marker") + } result.Changes = connectionObj.Changes @@ -746,15 +694,11 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } db.Delete(connectionObj) - err = writemarkerEntity.SendToChan(ctx) - if err != nil { - return nil, common.NewError("write_marker_error", "Error redeeming the write marker") - } - Logger.Info("write_marker_redeemed", zap.String("alloc_id", allocationID), zap.String("allocation_root", writeMarker.AllocationRoot)) go allocation.DeleteConnectionObjEntry(connectionID) Logger.Info("[commit]"+commitOperation, zap.String("alloc_id", allocationID), + zap.String("allocation_root", writeMarker.AllocationRoot), zap.String("input", input), zap.Duration("get_alloc", elapsedAllocation), zap.Duration("get-lock", elapsedGetLock), @@ -1015,7 +959,7 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int } objectRef, err := reference.GetLimitedRefFieldsByLookupHash( - ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root", "thumbnail_filename"}) + ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"}) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) @@ -1084,7 +1028,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn return nil, common.NewError("invalid_parameters", "Invalid path") } fileRef, err := reference.GetLimitedRefFieldsByPath(ctx, connectionObj.AllocationID, path, - []string{"path", "name", "size", "hash", "validation_root", "fixed_merkle_root", "thumbnail_filename"}) + []string{"path", "name", "size", "hash", "validation_root", "fixed_merkle_root"}) if err != nil { Logger.Error("invalid_file", zap.Error(err)) diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go index 6887c401c..201ceda0c 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler_test.go @@ -5,6 +5,7 @@ import ( "database/sql/driver" "encoding/json" "fmt" + "log" "net/http" "net/http/httptest" "time" @@ -48,6 +49,7 @@ func TestDownloadFile(t *testing.T) { mockClientWallet = "{\"client_id\":\"9a566aa4f8e8c342fed97c8928040a21f21b8f574e5782c28568635ba9c75a85\",\"client_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"keys\":[{\"public_key\":\"40cd10039913ceabacf05a7c60e1ad69bb2964987bc50f77495e514dc451f907c3d8ebcdab20eedde9c8f39b9a1d66609a637352f318552fb69d4b3672516d1a\",\"private_key\":\"a3a88aad5d89cec28c6e37c2925560ce160ac14d2cdcf4a4654b2bb358fe7514\"}],\"mnemonics\":\"inside february piece turkey offer merry select combine tissue wave wet shift room afraid december gown mean brick speak grant gain become toy clown\",\"version\":\"1.0\",\"date_created\":\"2021-05-21 17:32:29.484657 +0545 +0545 m=+0.072791323\"}" mockOwnerWallet = "{\"client_id\":\"5d0229e0141071c1f88785b1faba4b612582f9d446b02e8d893f1e0d0ce92cdc\",\"client_key\":\"aefef5778906680360cf55bf462823367161520ad95ca183445a879a59c9bf0470b74e41fc12f2ee0ce9c19c4e77878d734226918672d089f561ecf1d5435720\",\"keys\":[{\"public_key\":\"aefef5778906680360cf55bf462823367161520ad95ca183445a879a59c9bf0470b74e41fc12f2ee0ce9c19c4e77878d734226918672d089f561ecf1d5435720\",\"private_key\":\"4f8af6fb1098a3817d705aef96db933f31755674b00a5d38bb2439c0a27b0117\"}],\"mnemonics\":\"erode transfer noble civil ridge cloth sentence gauge board wheel sight caution okay sand ranch ice frozen frown grape lion feed fox game zone\",\"version\":\"1.0\",\"date_created\":\"2021-09-04T14:11:06+01:00\"}" mockReadPrice = int64(0.1 * 1e10) + mockReadPriceFree = int64(0) mockWritePrice = int64(0.5 * 1e10) mockBigBalance = int64(10000 * 1e10) ) @@ -106,27 +108,11 @@ func TestDownloadFile(t *testing.T) { t *testing.T, req *http.Request, p parameters, - ) *marker.ReadMarker { - rm := &marker.ReadMarker{} - rm.ClientID = client.GetClientID() - rm.ClientPublicKey = client.GetClientPublicKey() - rm.BlobberID = p.inData.blobber.ID - rm.AllocationID = p.inData.allocationID - rm.OwnerID = mockOwner.ClientID - rm.Timestamp = now - // set another value to size - rm.ReadCounter = p.inData.numBlocks - rm.SessionRC = p.inData.numBlocks - err := rm.Sign() - require.NoError(t, err) - rmData, err := json.Marshal(rm) - require.NoError(t, err) + ) { req.Header.Set("X-Path-Hash", p.inData.pathHash) req.Header.Set("X-Path", p.inData.remotefilepath) req.Header.Set("X-Block-Num", fmt.Sprintf("%d", p.inData.blockNum)) req.Header.Set("X-Num-Blocks", fmt.Sprintf("%d", p.inData.numBlocks)) - req.Header.Set("X-Submit-RM", fmt.Sprint(true)) - req.Header.Set("X-Read-Marker", string(rmData)) req.Header.Set(common.AllocationIdHeader, mockAllocationId) if p.useAuthTicket { @@ -147,7 +133,6 @@ func TestDownloadFile(t *testing.T) { if len(p.inData.contentMode) > 0 { req.Header.Set("X-Mode", p.inData.contentMode) } - return rm } makeMockMakeSCRestAPICall := func(t *testing.T, p parameters) func(scAddress string, relativePath string, params map[string]string, chain *chain.Chain) ([]byte, error) { @@ -184,7 +169,6 @@ func TestDownloadFile(t *testing.T) { setupInMock := func( t *testing.T, p parameters, - rm marker.ReadMarker, ) { if p.isRepairer { mocket.Catcher.NewMock().OneTime().WithQuery( @@ -215,17 +199,31 @@ func TestDownloadFile(t *testing.T) { ) } - mocket.Catcher.NewMock().OneTime().WithQuery( - `SELECT * FROM "terms" WHERE`, - ).WithArgs( - "mock_allocation_id", - ).OneTime().WithReply( - []map[string]interface{}{{ - "blobber_id": mockBlobberId, - "read_price": mockReadPrice, - "write_price": mockWritePrice, - }}, - ) + if p.isFunded0Chain || p.isFundedBlobber { + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "terms" WHERE`, + ).WithArgs( + "mock_allocation_id", + ).OneTime().WithReply( + []map[string]interface{}{{ + "blobber_id": mockBlobberId, + "read_price": mockReadPriceFree, + "write_price": mockWritePrice, + }}, + ) + } else { + mocket.Catcher.NewMock().OneTime().WithQuery( + `SELECT * FROM "terms" WHERE`, + ).WithArgs( + "mock_allocation_id", + ).OneTime().WithReply( + []map[string]interface{}{{ + "blobber_id": mockBlobberId, + "read_price": mockReadPrice, + "write_price": mockWritePrice, + }}, + ) + } mocket.Catcher.NewMock().OneTime().WithQuery( `SELECT * FROM "reference_objects" WHERE`, @@ -282,29 +280,7 @@ func TestDownloadFile(t *testing.T) { setupOutMock := func( t *testing.T, p parameters, - rm marker.ReadMarker, ) { - - mocket.Catcher.NewMock().WithCallback(func(par1 string, args []driver.NamedValue) { - require.EqualValues(t, client.GetClientID(), args[0].Value) - require.EqualValues(t, mockAllocationId, args[1].Value) - require.EqualValues(t, client.GetClientPublicKey(), args[2].Value) - require.EqualValues(t, mockOwner.ClientID, args[3].Value) - require.EqualValues(t, now, args[4].Value) - require.EqualValues(t, p.inData.numBlocks, args[5].Value) - }).WithQuery(`INSERT INTO "read_markers"`).WithID(11) - - mocket.Catcher.NewMock().WithCallback(func(par1 string, args []driver.NamedValue) { - //require.EqualValues(t, p.payerId.ClientKey, args[0].Value) - require.EqualValues(t, client.GetClientPublicKey(), args[0].Value) - // require.EqualValues(t, mockBlobberId, args[1].Value) - require.EqualValues(t, mockAllocationId, args[1].Value) - require.EqualValues(t, mockOwner.ClientID, args[2].Value) - require.EqualValues(t, now, args[3].Value) - require.EqualValues(t, p.inData.numBlocks, args[4].Value) - require.EqualValues(t, p.payerId.ClientID, args[6].Value) - }).WithQuery(`UPDATE "read_markers" SET`).WithID(1) - mocket.Catcher.NewMock().WithQuery(`UPDATE "file_stats" SET`).WithID(1) } @@ -319,10 +295,10 @@ func TestDownloadFile(t *testing.T) { return ctx } - setupRequest := func(p parameters) (*http.Request, *marker.ReadMarker) { + setupRequest := func(p parameters) *http.Request { req := httptest.NewRequest(http.MethodGet, "/v1/file/download/", nil) - rm := addToForm(t, req, p) - return req, rm + addToForm(t, req, p) + return req } setupParams := func(p *parameters) { @@ -333,7 +309,7 @@ func TestDownloadFile(t *testing.T) { remotefilepath: mockRemoteFilePath, blockNum: mockBlockNumber, encryptedKey: mockEncryptKey, - contentMode: "", + contentMode: DownloadContentFull, numBlocks: 1, } if p.isRepairer { @@ -392,7 +368,7 @@ func TestDownloadFile(t *testing.T) { }, want: want{ err: true, - errMsg: "not_enough_tokens: pre-redeeming read marker: read_pre_redeem: not enough tokens in client's read pools associated with the allocation->blobber", + errMsg: "download_file: no download quota for ", }, }, { @@ -480,16 +456,17 @@ func TestDownloadFile(t *testing.T) { require.NoError(t, client.PopulateClient(mockClientWallet, "bls0chain")) } transaction.MakeSCRestAPICall = makeMockMakeSCRestAPICall(t, test.parameters) - request, rm := setupRequest(test.parameters) + request := setupRequest(test.parameters) datastore.MocketTheStore(t, mocketLogging) - setupInMock(t, test.parameters, *rm) - setupOutMock(t, test.parameters, *rm) + setupInMock(t, test.parameters) + setupOutMock(t, test.parameters) ctx := setupCtx(test.parameters) ctx = context.WithValue(ctx, constants.ContextKeyAllocationID, mockAllocationId) var sh StorageHandler _, err := sh.DownloadFile(ctx, request) + log.Println("==============", err) require.EqualValues(t, test.want.err, err != nil) if err != nil { diff --git a/code/go/0chain.net/blobbercore/readmarker/readmarker.go b/code/go/0chain.net/blobbercore/readmarker/readmarker.go index 41f178668..e49ec747a 100644 --- a/code/go/0chain.net/blobbercore/readmarker/readmarker.go +++ b/code/go/0chain.net/blobbercore/readmarker/readmarker.go @@ -102,10 +102,10 @@ func (rm *ReadMarkerEntity) VerifyMarker(ctx context.Context, sa *allocation.All if clientID == "" || clientID != rm.LatestRM.ClientID { return common.NewError("read_marker_validation_failed", "Read Marker clientID does not match request clientID") } - currentTS := common.Now() - if rm.LatestRM.Timestamp > currentTS { - zLogger.Logger.Error("Timestamp is for future in the read marker", zap.Any("rm", rm), zap.Any("now", currentTS)) - return common.NewError("read_marker_validation_failed", "Timestamp is for future in the read marker") + + if rm.LatestRM.Timestamp > sa.Expiration { + zLogger.Logger.Error("Readmarker is for an expired allocation", zap.Any("rm", rm)) + return common.NewError("read_marker_validation_failed", "Readmarker is for an expired allocation") } hashData := rm.LatestRM.GetHashData() diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go index 24425c5d1..cc788a5e0 100644 --- a/code/go/0chain.net/blobbercore/reference/object.go +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -31,6 +31,7 @@ func DeleteObject(ctx context.Context, rootRef *Ref, allocationID, objPath strin rootRef.Children = nil rootRef.HashToBeComputed = true rootRef.childrenLoaded = true + rootRef.UpdatedAt = ts return nil } parentPath, deleteFileName := filepath.Split(objPath) diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 2b2c3ffbf..aab633054 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -12,6 +12,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -30,10 +32,9 @@ type Ref struct { ID int64 `gorm:"column:id;primaryKey"` FileID string `gorm:"column:file_id" dirlist:"file_id" filelist:"file_id"` Type string `gorm:"column:type;size:1" dirlist:"type" filelist:"type"` - AllocationID string `gorm:"column:allocation_id;size:64;not null;index:idx_path_alloc,priority:1;index:idx_lookup_hash_alloc,priority:1" dirlist:"allocation_id" filelist:"allocation_id"` + AllocationID string `gorm:"column:allocation_id;size:64;not null;index:idx_path_alloc,priority:1;index:idx_lookup_hash_alloc,priority:1;index:idx_validation_alloc,priority:1" dirlist:"allocation_id" filelist:"allocation_id"` LookupHash string `gorm:"column:lookup_hash;size:64;not null;index:idx_lookup_hash_alloc,priority:2" dirlist:"lookup_hash" filelist:"lookup_hash"` Name string `gorm:"column:name;size:100;not null;index:idx_name_gin:gin" dirlist:"name" filelist:"name"` - ThumbnailFilename string `gorm:"column:thumbnail_filename" dirlist:"thumbnail_filename" filelist:"thumbnail_filename"` Path string `gorm:"column:path;size:1000;not null;index:idx_path_alloc,priority:2;index:path_idx" dirlist:"path" filelist:"path"` FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"` Hash string `gorm:"column:hash;size:64;not null" dirlist:"hash" filelist:"hash"` @@ -42,7 +43,7 @@ type Ref struct { ParentPath string `gorm:"column:parent_path;size:999"` PathLevel int `gorm:"column:level;not null;default:0"` CustomMeta string `gorm:"column:custom_meta;not null" filelist:"custom_meta"` - ValidationRoot string `gorm:"column:validation_root;size:64;not null" filelist:"validation_root"` + ValidationRoot string `gorm:"column:validation_root;size:64;not null;index:idx_validation_alloc,priority:2" filelist:"validation_root"` PrevValidationRoot string `gorm:"column:prev_validation_root" filelist:"prev_validation_root" json:"prev_validation_root"` ValidationRootSignature string `gorm:"column:validation_root_signature;size:64" filelist:"validation_root_signature" json:"validation_root_signature,omitempty"` Size int64 `gorm:"column:size;not null;default:0" dirlist:"size" filelist:"size"` @@ -370,8 +371,8 @@ func GetRefWithSortedChildren(ctx context.Context, allocationID, path string) (* func (r *Ref) GetFileMetaHashData() string { return fmt.Sprintf( - "%s:%d:%s:%d:%s", - r.Path, r.Size, r.FileID, + "%s:%d:%d:%s", + r.Path, r.Size, r.ActualFileSize, r.ActualFileHash) } @@ -447,7 +448,7 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool) (h string, er actualSize += childRef.ActualFileSize } - r.FileMetaHash = encryption.Hash(r.GetHashData() + strings.Join(childFileMetaHashes, ":")) + r.FileMetaHash = encryption.Hash(r.Path + strings.Join(childFileMetaHashes, ":")) r.Hash = encryption.Hash(r.GetHashData() + strings.Join(childHashes, ":")) r.PathHash = encryption.Hash(strings.Join(childPathHashes, ":")) r.NumBlocks = refNumBlocks @@ -479,6 +480,9 @@ func (r *Ref) AddChild(child *Ref) { return } + if child.ParentPath != ref.ParentPath { + logging.Logger.Error("invalid parent path", zap.String("child", child.Path), zap.String("parent", ref.Path)) + } if strings.Compare(child.Path, ref.Path) == -1 { index = i ltFound = true @@ -599,7 +603,6 @@ func ListingDataToRef(refMap map[string]interface{}) *Ref { if len(refMap) < 1 { return nil } - ref := &Ref{} refType, _ := refMap["type"].(string) diff --git a/code/go/0chain.net/blobbercore/writemarker/protocol.go b/code/go/0chain.net/blobbercore/writemarker/protocol.go index 5bb8e1543..b11c82366 100644 --- a/code/go/0chain.net/blobbercore/writemarker/protocol.go +++ b/code/go/0chain.net/blobbercore/writemarker/protocol.go @@ -82,6 +82,12 @@ func (wme *WriteMarkerEntity) VerifyMarker(ctx context.Context, dbAllocation *al return common.NewError("write_marker_validation_failed", "Write Marker is not by the same client who uploaded") } + currTime := common.Now() + // blobber clock is allowed to be 10 seconds behing the current time + if wme.WM.Timestamp > currTime+10 { + return common.NewError("write_marker_validation_failed", "Write Marker timestamp is in the future") + } + hashData := wme.WM.GetHashData() signatureHash := encryption.Hash(hashData) sigOK, err := encryption.Verify(clientPublicKey, wme.WM.Signature, signatureHash) @@ -180,6 +186,10 @@ func (wme *WriteMarkerEntity) VerifyRollbackMarker(ctx context.Context, dbAlloca return common.NewError("write_marker_validation_failed", fmt.Sprintf("Write Marker allocation root %v does not match the previous allocation root of latest write marker %v", wme.WM.AllocationRoot, latestWM.WM.PreviousAllocationRoot)) } + if wme.WM.Timestamp != latestWM.WM.Timestamp { + return common.NewError("write_marker_validation_failed", fmt.Sprintf("Write Marker timestamp %v does not match the timestamp of latest write marker %v", wme.WM.Timestamp, latestWM.WM.Timestamp)) + } + clientPublicKey := ctx.Value(constants.ContextKeyClientKey).(string) if clientPublicKey == "" { return common.NewError("write_marker_validation_failed", "Could not get the public key of the client") diff --git a/code/go/0chain.net/blobbercore/writemarker/worker.go b/code/go/0chain.net/blobbercore/writemarker/worker.go index 833396336..da82e3fa2 100644 --- a/code/go/0chain.net/blobbercore/writemarker/worker.go +++ b/code/go/0chain.net/blobbercore/writemarker/worker.go @@ -99,7 +99,7 @@ func redeemWriteMarker(wm *WriteMarkerEntity) error { elapsedTime := time.Since(start) logging.Logger.Error("Error redeeming the write marker.", zap.Any("allocation", allocationID), - zap.Any("wm", wm.WM.AllocationID), zap.Any("error", err), zap.Any("elapsedTime", elapsedTime)) + zap.Any("wm", wm), zap.Any("error", err), zap.Any("elapsedTime", elapsedTime)) go tryAgain(wm) shouldRollback = true diff --git a/code/go/0chain.net/core/logging/logger.go b/code/go/0chain.net/core/logging/logger.go index 1e73d9571..fa29cf43c 100644 --- a/code/go/0chain.net/core/logging/logger.go +++ b/code/go/0chain.net/core/logging/logger.go @@ -2,6 +2,7 @@ package logging import ( "os" + "sync" "github.com/spf13/viper" "go.uber.org/zap" @@ -11,9 +12,16 @@ import ( var ( Logger *zap.Logger + once sync.Once ) func InitLogging(mode, logDir, logFile string) { + once.Do(func() { + initLogging(mode, logDir, logFile) + }) +} + +func initLogging(mode, logDir, logFile string) { var logName = logDir + "/" + logFile var logWriter = getWriteSyncer(logName) @@ -29,15 +37,27 @@ func InitLogging(mode, logDir, logFile string) { cfg.EncoderConfig.MessageKey = "msg" cfg.EncoderConfig.CallerKey = "caller" cfg.EncoderConfig.StacktraceKey = "stacktrace" - - logWriter = zapcore.NewMultiWriteSyncer(zapcore.AddSync(os.Stdout), logWriter) + if viper.GetBool("logging.console") { + logWriter = zapcore.NewMultiWriteSyncer(zapcore.AddSync(os.Stdout), logWriter) + } } - _ = cfg.Level.UnmarshalText([]byte(viper.GetString("logging.level"))) + if err := cfg.Level.UnmarshalText([]byte(viper.GetString("logging.level"))); err != nil { + panic(err) + } + cfg.Encoding = "console" cfg.EncoderConfig.TimeKey = "timestamp" cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - l, err := cfg.Build(SetOutput(logWriter, cfg)) + mlcfg := zap.NewProductionConfig() + if mode != "development" { + mlcfg.Level.SetLevel(zapcore.ErrorLevel) + } else { + mlcfg.Level.SetLevel(zapcore.DebugLevel) + } + + option := createOptionFromCores(createZapCore(logWriter, cfg)) + l, err := cfg.Build(option) if err != nil { panic(err) } @@ -45,6 +65,30 @@ func InitLogging(mode, logDir, logFile string) { Logger = l } +func createZapCore(ws zapcore.WriteSyncer, conf zap.Config) zapcore.Core { + enc := getEncoder(conf) + return zapcore.NewCore(enc, ws, conf.Level) +} + +func createOptionFromCores(cores ...zapcore.Core) zap.Option { + return zap.WrapCore(func(core zapcore.Core) zapcore.Core { + return zapcore.NewTee(cores...) + }) +} + +func getEncoder(conf zap.Config) zapcore.Encoder { + var enc zapcore.Encoder + switch conf.Encoding { + case "json": + enc = zapcore.NewJSONEncoder(conf.EncoderConfig) + case "console": + enc = zapcore.NewConsoleEncoder(conf.EncoderConfig) + default: + panic("unknown encoding") + } + return enc +} + // SetOutput replaces existing Core with new, that writes to passed WriteSyncer. func SetOutput(ws zapcore.WriteSyncer, conf zap.Config) zap.Option { var enc zapcore.Encoder diff --git a/code/go/0chain.net/core/transaction/nonce.go b/code/go/0chain.net/core/transaction/nonce.go index c21af0453..44013c7a6 100644 --- a/code/go/0chain.net/core/transaction/nonce.go +++ b/code/go/0chain.net/core/transaction/nonce.go @@ -56,7 +56,7 @@ func (m *nonceMonitor) recordFailedNonce(nonce int64) { m.failed[nonce]++ // when failing for same nonce often, should reschedule nonce for refresh from balance. - if m.failed[nonce] > 0 { + if m.failed[nonce] >= 3 { m.shouldRefreshFromBalance = true logging.Logger.Info("Frequent failures at nonce.", zap.Any("nonce", nonce), zap.Any("highestSuccess", m.highestSuccess)) } @@ -95,26 +95,19 @@ func (m *nonceMonitor) refreshFromBalance() { <-cb.waitCh + newNonce := int64(0) if cb.hasError { - return + logging.Logger.Info("Couldn't get nonce from remote, use 0") + newNonce = int64(0) + } else { + logging.Logger.Info("Got nonce from balance.", zap.Any("nonce", cb.nonce), zap.Any("highestSuccess", m.highestSuccess)) + newNonce = cb.nonce } - logging.Logger.Info("Got nonce from balance.", zap.Any("nonce", cb.nonce), zap.Any("highestSuccess", m.highestSuccess)) - - newNonce := cb.nonce - - oldHighest := m.highestSuccess m.highestSuccess = newNonce - // no clean up necessary if failed is empty - if len(m.failed) == 0 { - return - } - - // (clean up) delete entries on failed up to this new highest success - for i := oldHighest; i <= m.highestSuccess; i++ { - delete(m.failed, i) - } + m.failed = make(map[int64]int64) + m.used = make(map[int64]time.Time) } type getNonceCallBack struct { diff --git a/code/go/0chain.net/validator/main.go b/code/go/0chain.net/validator/main.go index 180fafab8..72bcf5c24 100644 --- a/code/go/0chain.net/validator/main.go +++ b/code/go/0chain.net/validator/main.go @@ -137,7 +137,6 @@ func main() { fmt.Println("*== Validator Wallet Info ==*") fmt.Println(" ID: ", node.Self.ID) fmt.Println(" Public Key: ", publicKey) - fmt.Println(" Private Key: ", privateKey) fmt.Println("*===========================*") chain.SetServerChain(serverChain) diff --git a/code/go/0chain.net/validatorcore/storage/challenge_handler.go b/code/go/0chain.net/validatorcore/storage/challenge_handler.go index 68ef44b6f..43c5de15c 100644 --- a/code/go/0chain.net/validatorcore/storage/challenge_handler.go +++ b/code/go/0chain.net/validatorcore/storage/challenge_handler.go @@ -77,11 +77,6 @@ func NewChallengeRequest(r *http.Request) (*ChallengeRequest, string, error) { return nil, "", common.NewError("invalid_parameters", "Header hash and request hash do not match") } - if challengeRequest.ObjPath == nil { - logging.Logger.Error("Not object path found in the input") - return nil, "", common.NewError("invalid_parameters", "Empty object path or merkle path") - } - return &challengeRequest, challengeHash, err } diff --git a/code/go/0chain.net/validatorcore/storage/challenge_handler_test.go b/code/go/0chain.net/validatorcore/storage/challenge_handler_test.go index 58f933583..92de8f023 100644 --- a/code/go/0chain.net/validatorcore/storage/challenge_handler_test.go +++ b/code/go/0chain.net/validatorcore/storage/challenge_handler_test.go @@ -55,16 +55,6 @@ func TestChallengeHandler(t *testing.T) { wantErr: true, wantErrMsg: "Header hash and request hash do not match", }, - { - name: "nil object path", - req: func() *http.Request { - req, _ := http.NewRequest("POST", "url", bytes.NewBuffer([]byte("{}"))) - req.Header.Set("X-App-Request-Hash", "840eb7aa2a9935de63366bacbe9d97e978a859e93dc792a0334de60ed52f8e99") - return req - }(), - wantErr: true, - wantErrMsg: "Empty object path or merkle path", - }, } for _, tt := range tests { diff --git a/code/go/0chain.net/validatorcore/storage/models.go b/code/go/0chain.net/validatorcore/storage/models.go index 4ab65d001..a7dbe543e 100644 --- a/code/go/0chain.net/validatorcore/storage/models.go +++ b/code/go/0chain.net/validatorcore/storage/models.go @@ -297,9 +297,11 @@ func (cr *ChallengeRequest) verifyBlockNum(challengeObj *Challenge) error { func (cr *ChallengeRequest) VerifyChallenge(challengeObj *Challenge, allocationObj *Allocation) error { logging.Logger.Info("Verifying object path", zap.String("challenge_id", challengeObj.ID), zap.Int64("seed", challengeObj.RandomNumber)) - err := cr.ObjPath.Verify(challengeObj.AllocationID, challengeObj.RandomNumber) - if err != nil { - return common.NewError("challenge_validation_failed", "Failed to verify the object path."+err.Error()) + if cr.ObjPath != nil { + err := cr.ObjPath.Verify(challengeObj.AllocationID, challengeObj.RandomNumber) + if err != nil { + return common.NewError("challenge_validation_failed", "Failed to verify the object path."+err.Error()) + } } if len(cr.WriteMarkers) == 0 { @@ -307,7 +309,7 @@ func (cr *ChallengeRequest) VerifyChallenge(challengeObj *Challenge, allocationO } logging.Logger.Info("Verifying write marker", zap.String("challenge_id", challengeObj.ID)) - err = cr.WriteMarkers[0].WM.Verify(allocationObj.ID, challengeObj.AllocationRoot, cr.WriteMarkers[0].ClientPublicKey) + err := cr.WriteMarkers[0].WM.Verify(allocationObj.ID, challengeObj.AllocationRoot, cr.WriteMarkers[0].ClientPublicKey) if err != nil { return common.NewError("write_marker_validation_failed", "Failed to verify the write marker. "+err.Error()) } @@ -326,14 +328,21 @@ func (cr *ChallengeRequest) VerifyChallenge(challengeObj *Challenge, allocationO } } latestWM := cr.WriteMarkers[len(cr.WriteMarkers)-1].WM - rootRef := cr.ObjPath.RootObject - allocationRootCalculated := rootRef.Hash + if cr.ObjPath != nil { + rootRef := cr.ObjPath.RootObject + allocationRootCalculated := rootRef.Hash - if latestWM.AllocationRoot != allocationRootCalculated { - return common.NewError("challenge_validation_failed", "Allocation root does not match") - } + if latestWM.AllocationRoot != allocationRootCalculated { + return common.NewError("challenge_validation_failed", "Allocation root does not match") + } - if rootRef.NumBlocks == 0 { + if rootRef.NumBlocks == 0 { + return nil + } + } else { + if latestWM.AllocationRoot != "" || latestWM.PreviousAllocationRoot != "" { + return common.NewError("challenge_validation_failed", "Allocation root is not empty") + } return nil } diff --git a/docker.local/.dockerignore b/docker.local/.dockerignore index 5691e851e..628d1ada6 100644 --- a/docker.local/.dockerignore +++ b/docker.local/.dockerignore @@ -2,4 +2,7 @@ passphrase.txt logs/ .git *.md -.cache \ No newline at end of file +.cache +awsnet +docker.local/blobber* +**/pkg \ No newline at end of file diff --git a/docker.local/b0docker-compose.yml b/docker.local/b0docker-compose.yml index 354ca67be..fee51770a 100644 --- a/docker.local/b0docker-compose.yml +++ b/docker.local/b0docker-compose.yml @@ -36,7 +36,7 @@ services: links: - postgres-post:postgres-post volumes: - - ../config:/validator/config + - ${CONFIG_PATH:-../config}:/validator/config #value after :- is default value - ./blobber${BLOBBER}/data:/validator/data - ./blobber${BLOBBER}/log:/validator/log - ./keys_config:/validator/keysconfig @@ -57,7 +57,7 @@ services: links: - validator:validator volumes: - - ../config:/blobber/config + - ${CONFIG_PATH:-../config}:/blobber/config - ./blobber${BLOBBER}/files:/blobber/files - ./blobber${BLOBBER}/data:/blobber/data - ./blobber${BLOBBER}/log:/blobber/log diff --git a/docker.local/conductor-b0docker-compose.yml b/docker.local/conductor-b0docker-compose.yml deleted file mode 100644 index b23d08550..000000000 --- a/docker.local/conductor-b0docker-compose.yml +++ /dev/null @@ -1,89 +0,0 @@ -version: "3" -services: - postgres: - image: postgres:14 - environment: - POSTGRES_PORT: 5432 - POSTGRES_HOST: postgres - POSTGRES_USER: postgres - POSTGRES_PASSWORD: secret - POSTGRES_HOST_AUTH_METHOD: trust -#expose_ci_port - volumes: - - ./blobber${BLOBBER}/data/postgresql:/var/lib/postgresql/data - networks: - default: - postgres-post: - image: postgres:14 - environment: - POSTGRES_PORT: 5432 - POSTGRES_HOST: postgres - POSTGRES_USER: postgres - POSTGRES_PASSWORD: secret - volumes: - - ../bin:/blobber/bin - - ../sql:/blobber/sql - command: bash /blobber/bin/postgres-entrypoint.sh - links: - - postgres:postgres - validator: - image: validator - environment: - - DOCKER= true - depends_on: - - postgres-post - links: - - postgres-post:postgres-post - volumes: - - ./conductor-config:/validator/config - - ./blobber${BLOBBER}/data:/validator/data - - ./blobber${BLOBBER}/log:/validator/log - - ./keys_config:/validator/keysconfig - ports: - - "506${BLOBBER}:506${BLOBBER}" - command: ./bin/validator --port 506${BLOBBER} --hostname 198.18.0.6${BLOBBER} --deployment_mode 0 --keys_file keysconfig/b0vnode${BLOBBER}_keys.txt --log_dir /validator/log - networks: - default: - testnet0: - ipv4_address: 198.18.0.6${BLOBBER} - - blobber: - image: blobber - environment: - - DOCKER= true - depends_on: - - validator - links: - - validator:validator - volumes: - - ./conductor-config:/blobber/config - - ./blobber${BLOBBER}/files:/blobber/files - - ./blobber${BLOBBER}/data:/blobber/data - - ./blobber${BLOBBER}/log:/blobber/log - - ./keys_config:/blobber/keysconfig - - ./blobber${BLOBBER}/data/tmp:/tmp - ports: - - "505${BLOBBER}:505${BLOBBER}" - - "3150${BLOBBER}:3150${BLOBBER}" - command: ./bin/blobber --port 505${BLOBBER} --grpc_port 3150${BLOBBER} --hostname 198.18.0.9${BLOBBER} --deployment_mode 0 --keys_file keysconfig/b0bnode${BLOBBER}_keys.txt --files_dir /blobber/files --log_dir /blobber/log --db_dir /blobber/data - networks: - default: - testnet0: - ipv4_address: 198.18.0.9${BLOBBER} - -networks: - default: - driver: bridge - testnet0: - external: true - # driver: bridge - # ipam: - # config: - # - subnet: 198.18.0.0/15 - # - gateway: 198.18.0.255 - - -volumes: - data: - config: - bin: diff --git a/go.mod b/go.mod index 0ece48c52..d2b1f2958 100644 --- a/go.mod +++ b/go.mod @@ -21,33 +21,30 @@ require ( github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.9.0 + golang.org/x/crypto v0.10.0 golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 + golang.org/x/sys v0.9.0 golang.org/x/time v0.3.0 // indirect google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect - google.golang.org/grpc v1.55.0 + google.golang.org/grpc v1.56.1 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 - google.golang.org/protobuf v1.30.0 + google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gorm.io/datatypes v1.2.0 gorm.io/driver/postgres v1.5.2 gorm.io/driver/sqlite v1.5.2 - gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 + gorm.io/gorm v1.25.2 ) require ( github.com/lithammer/shortuuid/v3 v3.0.7 - golang.org/x/sync v0.2.0 + golang.org/x/sync v0.3.0 google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc ) require google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect -require ( - github.com/lithammer/shortuuid/v3 v3.0.7 - github.com/pressly/goose/v3 v3.11.2 -) +require github.com/pressly/goose/v3 v3.13.1 require ( github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 // indirect @@ -88,11 +85,11 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.3.1 // indirect + github.com/jackc/pgx/v5 v5.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/compress v1.16.6 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/klauspost/reedsolomon v1.11.7 // indirect github.com/machinebox/graphql v0.2.2 // indirect @@ -123,7 +120,7 @@ require ( go.mongodb.org/mongo-driver v1.11.3 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect diff --git a/go.sum b/go.sum index 40633020d..10a27165f 100644 --- a/go.sum +++ b/go.sum @@ -435,8 +435,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.3.1 h1:Fcr8QJ1ZeLi5zsPZqQeUZhNhxfkkKBOgJuYkJHoBOtU= -github.com/jackc/pgx/v5 v5.3.1/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= +github.com/jackc/pgx/v5 v5.4.0 h1:BSr+GCm4N6QcgIwv0DyTFHK9ugfEFF9DzSbbzxOiXU0= +github.com/jackc/pgx/v5 v5.4.0/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -473,8 +473,8 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= +github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/reedsolomon v1.11.7 h1:9uaHU0slncktTEEg4+7Vl7q7XUNMBUOK4R9gnKhMjAU= @@ -524,7 +524,7 @@ github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/microsoft/go-mssqldb v0.21.0 h1:p2rpHIL7TlSv1QrbXJUAcbyRKnIT0C9rRkH2E4OjLn8= +github.com/microsoft/go-mssqldb v1.1.0 h1:jsV+tpvcPTbNNKW0o3kiCD69kOHICsfjZ2VcVu2lKYc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -604,8 +604,8 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pressly/goose/v3 v3.11.2 h1:QgTP45FhBBHdmf7hWKlbWFHtwPtxo0phSDkwDKGUrYs= -github.com/pressly/goose/v3 v3.11.2/go.mod h1:LWQzSc4vwfHA/3B8getTp8g3J5Z8tFBxgxinmGlMlJk= +github.com/pressly/goose/v3 v3.13.1 h1:4kYsfsboJvQDg1MEcMnXs8ubbz4AeHHXMW19XEAy3vI= +github.com/pressly/goose/v3 v3.13.1/go.mod h1:1WCIzQuOGhpx+D7bal+tGadbwVPmdfy9fltqYLMA8lM= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -631,7 +631,7 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= @@ -798,8 +798,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -835,8 +835,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -906,8 +905,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -975,8 +974,8 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -987,8 +986,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1057,8 +1056,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1158,8 +1156,8 @@ google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1174,8 +1172,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1222,8 +1220,8 @@ gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc= gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 h1:sC1Xj4TYrLqg1n3AN10w871An7wJM0gzgcm8jkIkECQ= -gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= +gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1233,13 +1231,13 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= -modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= -modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= -modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= +modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q= +modernc.org/ccgo/v3 v3.16.14 h1:af6KNtFgsVmnDYrWk3PQCS9XT6BXe7o3ZFJKkIKvXNQ= +modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= -modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.6.0 h1:i6mzavxrE9a30whzMfwf7XWVODx2r5OYXvU46cirX7o= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/sqlite v1.22.1 h1:P2+Dhp5FR1RlVRkQ3dDfCiv3Ok8XPxqpe70IjYVA9oE= +modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= From 94bd5571ed398b4faab6327ac59a473874e58017 Mon Sep 17 00:00:00 2001 From: Manali-Jain-squareops <86873900+Manali-Jain-squareops@users.noreply.github.com> Date: Wed, 19 Jul 2023 15:50:06 +0530 Subject: [PATCH 2/4] build base blobber optimization (#1169) * Add path in thumbnail hash (#1098) * get path from changes * pass rootRef in apply changes * add logs * fix deletechange * fix multi op and copy * fix delete root dir * fix delete root change * fix lint issue * cleanup and stats change * rename func to processMove * calculate object tree in ref path * rmv return from deleteChange * revert changes * move to filestore changes * check prevRoot * add logs and condition * check filestore * requested changes * validate nil proof * add move and copy dir tests * add logs * use path to add child * fix error log * fix add child * add proof log * add more logs * change read log * log objectPath * add commit logs * commit thumbnail first * add missing param * add missing param * add path in the thumbnail hash * add allocID and validation root index * use write * add condn in count query * rmv thumbnail_filename * log wm * cleanup * add rollback wm check * allow empty allocation root * fix tests * check path in upload * fix check * empty commit --------- Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Co-authored-by: Yury * Dep/update (#1153) * dependencies update * dependencies update * Remove fileID from fileMetaHash (#1114) * rmv fileID from hash calc * Trigger Build * adding false commit to restart systemtests * add path in fileMetaHash * empty commit --------- Co-authored-by: Yury Co-authored-by: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> Co-authored-by: boddumanohar Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> * optimize image (#1148) Co-authored-by: boddumanohar * once for logger init * Update challenge timing submission (#1140) * update challenge timing submission * fix createdAt in challenge timing table --------- Co-authored-by: Yury * Fix blobber size (#1163) * build base blobber optimization * fixed indentation * fix * fix * fix * Do not redeem readmarkers for free reads (#1166) * do not redeem readmarkers for free reads * fix unit tests, remove readmarker handling from download method * added docker ignore * added docker ignore - testing * once for logger init (#1156) * once for logger init * init logging the same way we do in 0chain * init logging the same way we do in 0chain * Hotfix/map concurrent write (#1158) * once for logger init * wider locking window * Hotfix/remove custom nonce (#1141) * removed custom nonce managing logic * fixed logging * updated gosdk * uncommented previous nonce logic * uncommented previous nonce logic * uncommented previous nonce logic * uncommented previous nonce logic * Use single file to avoid maintaining multiple version of same file (#1160) * Use single file to avoid maintaining multiple version of same file * Fix config path * Remove variable * Update path variable for blobber service --------- Co-authored-by: Yury * fix consume quota lock (#1173) * Fix rm and wm timestamp (#1162) * once for logger init * init logging the same way we do in 0chain * init logging the same way we do in 0chain * add lastChallengeTime log * empty commit * refactor timestamp check in markers * Add diff txn for move to filestore * return if duplicate --------- Co-authored-by: dabasov * fixed gitactions fix issue * remove printing private key in logs (#1161) Co-authored-by: Yury * Fix concurrent upload issue (#1174) * Fix concurrent upload issue * Fix concurrent upload issue * Fix/concurrent upload issue (#1175) * Fix concurrent upload issue * Fix concurrent upload issue * Fix concurrent upload issue * Fix concurrent upload issue --------- Co-authored-by: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Co-authored-by: Yury Co-authored-by: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> Co-authored-by: boddumanohar Co-authored-by: Jayash Satolia <73050737+Jayashsatolia403@users.noreply.github.com> Co-authored-by: stewartie4 Co-authored-by: Dinmukhammed <52813950+din-mukhammed@users.noreply.github.com> Co-authored-by: Laxmi Prasad Oli Co-authored-by: root --- .../build-&-publish-docker-image.yml | 48 +++++++++++++++++-- docker.local/.dockerignore | 5 ++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 30d0aef92..9d78f2677 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -16,6 +16,8 @@ env: BLOBBER_REGISTRY: ${{ secrets.BLOBBER_REGISTRY }} VALIDATOR_REGISTRY: ${{ secrets.VALIDATOR_REGISTRY }} DOCKER_CLI_EXPERIMENTAL: enabled + BLOBBER_BUILDBASE: blobber_base + BLOBBER_BUILD_BASE_REGISTRY: ${{ secrets.BLOBBER_BUILD_BASE_REGISTRY }} jobs: blobber: @@ -39,7 +41,7 @@ jobs: go-version: ^1.20 # The Go version to download (if necessary) and use. - name: Clone blobber - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: Set up Docker Buildx run: | @@ -60,6 +62,26 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Get changed files using defaults + id: changed-files + uses: tj-actions/changed-files@v18.4 + + - name: Pull Build Base + run: | + docker pull $BLOBBER_BUILD_BASE_REGISTRY:staging + docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE + + - name: Build Base image + if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile') + run: | + SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) + + ./docker.local/bin/build.base.sh && + docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG + docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA + docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG + docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA + - name: Build blobber run: | SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) @@ -68,7 +90,7 @@ jobs: export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:${TAG} -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}" docker buildx create --driver-opt network=host --use --buildkitd-flags '--allow-insecure-entitlement security.insecure' --use blobber_buildx - ./docker.local/bin/build.base.sh && ./docker.local/bin/build.blobber.sh + ./docker.local/bin/build.blobber.sh validator: runs-on: [self-hosted, arc-runner] @@ -112,6 +134,26 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Get changed files using defaults + id: changed-files + uses: tj-actions/changed-files@v18.4 + + - name: Pull Build Base + run: | + docker pull $BLOBBER_BUILD_BASE_REGISTRY:staging + docker tag $BLOBBER_BUILD_BASE_REGISTRY:staging $BLOBBER_BUILDBASE + + - name: Build Base image + if: contains(steps.changed-files.outputs.modified_files, 'docker.local/base.Dockerfile') + run: | + SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) + + ./docker.local/bin/build.base.sh + docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG + docker tag $BLOBBER_BUILDBASE $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA + docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG + docker push $BLOBBER_BUILD_BASE_REGISTRY:$TAG-$SHORT_SHA + - name: Build validator run: | SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) @@ -119,7 +161,7 @@ jobs: export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" export DOCKER_IMAGE_VALIDATOR="-t ${VALIDATOR_REGISTRY}:${TAG} -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}" docker buildx create --driver-opt network=host --use --buildkitd-flags '--allow-insecure-entitlement security.insecure' --use blobber_buildx - ./docker.local/bin/build.base.sh && ./docker.local/bin/build.validator.sh + ./docker.local/bin/build.validator.sh system-tests: diff --git a/docker.local/.dockerignore b/docker.local/.dockerignore index 628d1ada6..55bfc88ca 100644 --- a/docker.local/.dockerignore +++ b/docker.local/.dockerignore @@ -1,8 +1,13 @@ passphrase.txt logs/ +docs/ .git *.md .cache awsnet docker.local/blobber* +docker.aws/* +config/ +docker-clean/ +keys_config **/pkg \ No newline at end of file From 32bc5de95d25b4ac34c90cef34a1ca9773ff38eb Mon Sep 17 00:00:00 2001 From: Yury Date: Sat, 22 Jul 2023 15:27:14 +0300 Subject: [PATCH 3/4] updated dependencies (#1172) * Add path in thumbnail hash (#1098) * get path from changes * pass rootRef in apply changes * add logs * fix deletechange * fix multi op and copy * fix delete root dir * fix delete root change * fix lint issue * cleanup and stats change * rename func to processMove * calculate object tree in ref path * rmv return from deleteChange * revert changes * move to filestore changes * check prevRoot * add logs and condition * check filestore * requested changes * validate nil proof * add move and copy dir tests * add logs * use path to add child * fix error log * fix add child * add proof log * add more logs * change read log * log objectPath * add commit logs * commit thumbnail first * add missing param * add missing param * add path in the thumbnail hash * add allocID and validation root index * use write * add condn in count query * rmv thumbnail_filename * log wm * cleanup * add rollback wm check * allow empty allocation root * fix tests * check path in upload * fix check * empty commit --------- Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Co-authored-by: Yury * Dep/update (#1153) * dependencies update * dependencies update * Remove fileID from fileMetaHash (#1114) * rmv fileID from hash calc * Trigger Build * adding false commit to restart systemtests * add path in fileMetaHash * empty commit --------- Co-authored-by: Yury Co-authored-by: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> Co-authored-by: boddumanohar Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> * optimize image (#1148) Co-authored-by: boddumanohar * once for logger init * Update challenge timing submission (#1140) * update challenge timing submission * fix createdAt in challenge timing table --------- Co-authored-by: Yury * Fix blobber size (#1163) * Do not redeem readmarkers for free reads (#1166) * do not redeem readmarkers for free reads * fix unit tests, remove readmarker handling from download method * updated dependencies * once for logger init (#1156) * once for logger init * init logging the same way we do in 0chain * init logging the same way we do in 0chain * Hotfix/map concurrent write (#1158) * once for logger init * wider locking window * Hotfix/remove custom nonce (#1141) * removed custom nonce managing logic * fixed logging * updated gosdk * uncommented previous nonce logic * uncommented previous nonce logic * uncommented previous nonce logic * uncommented previous nonce logic * Use single file to avoid maintaining multiple version of same file (#1160) * Use single file to avoid maintaining multiple version of same file * Fix config path * Remove variable * Update path variable for blobber service --------- Co-authored-by: Yury * fix consume quota lock (#1173) * Fix rm and wm timestamp (#1162) * once for logger init * init logging the same way we do in 0chain * init logging the same way we do in 0chain * add lastChallengeTime log * empty commit * refactor timestamp check in markers * Add diff txn for move to filestore * return if duplicate --------- Co-authored-by: dabasov * fixed gitactions fix issue * remove printing private key in logs (#1161) Co-authored-by: Yury * Fix concurrent upload issue (#1174) * Fix concurrent upload issue * Fix concurrent upload issue * Update go.mod --------- Co-authored-by: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Co-authored-by: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Co-authored-by: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> Co-authored-by: boddumanohar Co-authored-by: Manali-Jain-squareops <86873900+Manali-Jain-squareops@users.noreply.github.com> Co-authored-by: Jayash Satolia <73050737+Jayashsatolia403@users.noreply.github.com> Co-authored-by: stewartie4 Co-authored-by: Dinmukhammed <52813950+din-mukhammed@users.noreply.github.com> Co-authored-by: Laxmi Prasad Oli Co-authored-by: root --- go.mod | 16 ++++++++-------- go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index d2b1f2958..df39f01df 100644 --- a/go.mod +++ b/go.mod @@ -21,12 +21,12 @@ require ( github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.10.0 + golang.org/x/crypto v0.11.0 golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.9.0 + golang.org/x/sys v0.10.0 golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect - google.golang.org/grpc v1.56.1 + google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect + google.golang.org/grpc v1.56.2 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -44,7 +44,7 @@ require ( require google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect -require github.com/pressly/goose/v3 v3.13.1 +require github.com/pressly/goose/v3 v3.13.4 require ( github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 // indirect @@ -85,11 +85,11 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.4.0 // indirect + github.com/jackc/pgx/v5 v5.4.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.16.6 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/klauspost/reedsolomon v1.11.7 // indirect github.com/machinebox/graphql v0.2.2 // indirect @@ -120,7 +120,7 @@ require ( go.mongodb.org/mongo-driver v1.11.3 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/text v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect diff --git a/go.sum b/go.sum index 10a27165f..745928389 100644 --- a/go.sum +++ b/go.sum @@ -435,8 +435,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.4.0 h1:BSr+GCm4N6QcgIwv0DyTFHK9ugfEFF9DzSbbzxOiXU0= -github.com/jackc/pgx/v5 v5.4.0/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= +github.com/jackc/pgx/v5 v5.4.1 h1:oKfB/FhuVtit1bBM3zNRRsZ925ZkMN3HXL+LgLUM9lE= +github.com/jackc/pgx/v5 v5.4.1/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -473,8 +473,8 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= -github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/reedsolomon v1.11.7 h1:9uaHU0slncktTEEg4+7Vl7q7XUNMBUOK4R9gnKhMjAU= @@ -524,7 +524,7 @@ github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/microsoft/go-mssqldb v1.1.0 h1:jsV+tpvcPTbNNKW0o3kiCD69kOHICsfjZ2VcVu2lKYc= +github.com/microsoft/go-mssqldb v1.3.0 h1:JcPVl+acL8Z/cQcJc9zP0OkjQ+l20bco/cCDpMbmGJk= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -604,8 +604,8 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pressly/goose/v3 v3.13.1 h1:4kYsfsboJvQDg1MEcMnXs8ubbz4AeHHXMW19XEAy3vI= -github.com/pressly/goose/v3 v3.13.1/go.mod h1:1WCIzQuOGhpx+D7bal+tGadbwVPmdfy9fltqYLMA8lM= +github.com/pressly/goose/v3 v3.13.4 h1:9xRcg/hEU9HqeRNeKh69VLtPWCKAYTX6l2VsXWOX86A= +github.com/pressly/goose/v3 v3.13.4/go.mod h1:Fo8rYaf9tYfQiDpo+ymrnZi8vvLkvguRl16nu7QnUT4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -631,7 +631,7 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= @@ -798,8 +798,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -835,7 +835,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -974,8 +974,8 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -986,8 +986,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1156,8 +1156,8 @@ google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= -google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= +google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1234,7 +1234,7 @@ lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q= modernc.org/ccgo/v3 v3.16.14 h1:af6KNtFgsVmnDYrWk3PQCS9XT6BXe7o3ZFJKkIKvXNQ= modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= -modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/memory v1.6.0 h1:i6mzavxrE9a30whzMfwf7XWVODx2r5OYXvU46cirX7o= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= From 697233e06e04fe4856332180e5574d29838e7a71 Mon Sep 17 00:00:00 2001 From: Jayash Satolia <73050737+Jayashsatolia403@users.noreply.github.com> Date: Sun, 23 Jul 2023 21:11:34 +0530 Subject: [PATCH 4/4] Fix/get open challenges (#1181) * Fix * Fix * Fix * Fix * Fix * Fix * Fix * Improved logging * Fix challenge table --- .../blobbercore/challenge/challenge.go | 34 +++++++++++-------- .../blobbercore/challenge/entity.go | 5 +-- .../blobbercore/challenge/protocol.go | 2 +- .../blobbercore/challenge/worker.go | 8 +++-- goose/migrations/001_blobber_meta.sql | 1 + 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/code/go/0chain.net/blobbercore/challenge/challenge.go b/code/go/0chain.net/blobbercore/challenge/challenge.go index 3f77cb801..4bdea9cfb 100644 --- a/code/go/0chain.net/blobbercore/challenge/challenge.go +++ b/code/go/0chain.net/blobbercore/challenge/challenge.go @@ -24,25 +24,23 @@ type BCChallengeResponse struct { Challenges []*ChallengeEntity `json:"challenges"` } -var lastChallengeTimestamp int +var lastChallengeRound int64 func syncOpenChallenges(ctx context.Context) { - const incrOffset = 20 defer func() { if r := recover(); r != nil { logging.Logger.Error("[recover]challenge", zap.Any("err", r)) } }() - offset := 0 params := make(map[string]string) params["blobber"] = node.Self.ID - params["offset"] = strconv.Itoa(offset) - params["limit"] = "20" - if lastChallengeTimestamp > 0 { - params["from"] = strconv.Itoa(lastChallengeTimestamp) + + params["limit"] = "50" + if lastChallengeRound > 0 { + params["from"] = strconv.FormatInt(lastChallengeRound, 10) } - logging.Logger.Info("[challenge]sync:pull", zap.Any("params", params)) + start := time.Now() var downloadElapsed, jsonElapsed time.Duration @@ -54,6 +52,9 @@ func syncOpenChallenges(ctx context.Context) { return default: } + + logging.Logger.Info("[challenge]sync:pull", zap.Any("params", params)) + var challenges BCChallengeResponse var challengeIDs []string challenges.Challenges = make([]*ChallengeEntity, 0) @@ -75,13 +76,13 @@ func syncOpenChallenges(ctx context.Context) { break } sort.Slice(challenges.Challenges, func(i, j int) bool { - return challenges.Challenges[i].CreatedAt < challenges.Challenges[j].CreatedAt + return challenges.Challenges[i].RoundCreatedAt < challenges.Challenges[j].RoundCreatedAt }) count += len(challenges.Challenges) for _, c := range challenges.Challenges { challengeIDs = append(challengeIDs, c.ChallengeID) - if c.CreatedAt > common.Timestamp(lastChallengeTimestamp) { - lastChallengeTimestamp = int(c.CreatedAt) + if c.RoundCreatedAt >= lastChallengeRound { + lastChallengeRound = c.RoundCreatedAt } toProcessChallenge <- c } @@ -93,8 +94,6 @@ func syncOpenChallenges(ctx context.Context) { if len(challenges.Challenges) == 0 { break } - offset += incrOffset - params["offset"] = strconv.Itoa(offset) } dbTimeStart := time.Now() @@ -110,6 +109,11 @@ func syncOpenChallenges(ctx context.Context) { func validateOnValidators(c *ChallengeEntity) { + logging.Logger.Info("[challenge]validate: ", + zap.Any("challenge", c), + zap.String("challenge_id", c.ChallengeID), + ) + ctx := datastore.GetStore().CreateTransaction(context.TODO()) defer ctx.Done() @@ -119,7 +123,7 @@ func validateOnValidators(c *ChallengeEntity) { logging.Logger.Error("[challengetiming]add: ", zap.String("challenge_id", c.ChallengeID), zap.Error(err)) - deleteChallenge(int64(c.CreatedAt)) + deleteChallenge(c.RoundCreatedAt) tx.Rollback() } @@ -150,7 +154,7 @@ func validateOnValidators(c *ChallengeEntity) { zap.Time("created", createdTime), zap.Error(err)) //TODO: Should we delete the challenge from map or send it back to the todo channel? - deleteChallenge(int64(c.CreatedAt)) + deleteChallenge(c.RoundCreatedAt) tx.Rollback() return } diff --git a/code/go/0chain.net/blobbercore/challenge/entity.go b/code/go/0chain.net/blobbercore/challenge/entity.go index 30eb0bf39..d30fb28f2 100644 --- a/code/go/0chain.net/blobbercore/challenge/entity.go +++ b/code/go/0chain.net/blobbercore/challenge/entity.go @@ -97,8 +97,9 @@ type ChallengeEntity struct { Timestamp common.Timestamp `gorm:"column:timestamp;not null;default:0" json:"timestamp"` // This time is taken from Blockchain challenge object. - CreatedAt common.Timestamp `gorm:"created_at" json:"created"` - UpdatedAt time.Time `gorm:"updated_at;type:timestamp without time zone;not null;default:current_timestamp" json:"-"` + RoundCreatedAt int64 `gorm:"round_created_at" json:"round_created_at"` + CreatedAt common.Timestamp `gorm:"created_at" json:"created"` + UpdatedAt time.Time `gorm:"updated_at;type:timestamp without time zone;not null;default:current_timestamp" json:"-"` } func (ChallengeEntity) TableName() string { diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index 1c297fdf3..67a853300 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -48,7 +48,7 @@ type ChallengeResponse struct { func (cr *ChallengeEntity) CancelChallenge(ctx context.Context, errReason error) { cancellation := time.Now() db := datastore.GetStore().GetTransaction(ctx) - deleteChallenge(int64(cr.CreatedAt)) + deleteChallenge(cr.RoundCreatedAt) cr.Status = Cancelled cr.StatusMessage = errReason.Error() cr.UpdatedAt = cancellation.UTC() diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go index 4afc61c56..4c2b01d6f 100644 --- a/code/go/0chain.net/blobbercore/challenge/worker.go +++ b/code/go/0chain.net/blobbercore/challenge/worker.go @@ -81,6 +81,8 @@ func challengeProcessor(ctx context.Context) { return case it := <-toProcessChallenge: + + logging.Logger.Info("processing_challenge", zap.Any("challenge_id", it.ChallengeID)) if ok := it.createChallenge(); !ok { continue } @@ -142,7 +144,7 @@ func commitOnChainWorker(ctx context.Context) { }() err := challenge.VerifyChallengeTransaction(txn) if err == nil || err != ErrEntityNotFound { - deleteChallenge(int64(challenge.CreatedAt)) + deleteChallenge(int64(challenge.RoundCreatedAt)) } }(&chall) } @@ -175,11 +177,11 @@ func getBatch(batchSize int) (chall []ChallengeEntity) { func (it *ChallengeEntity) createChallenge() bool { challengeMapLock.Lock() - if _, ok := challengeMap.Get(int64(it.CreatedAt)); ok { + if _, ok := challengeMap.Get(it.RoundCreatedAt); ok { challengeMapLock.Unlock() return false } - challengeMap.Put(int64(it.CreatedAt), it) + challengeMap.Put(it.RoundCreatedAt, it) challengeMapLock.Unlock() return true } diff --git a/goose/migrations/001_blobber_meta.sql b/goose/migrations/001_blobber_meta.sql index 7b5d72ec1..9eb80ae34 100644 --- a/goose/migrations/001_blobber_meta.sql +++ b/goose/migrations/001_blobber_meta.sql @@ -140,6 +140,7 @@ CREATE TABLE public.challenges ( object_path jsonb, sequence bigint, "timestamp" bigint DEFAULT 0 NOT NULL, + round_created_at bigint, created_at bigint, updated_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL );