Skip to content

Commit b05f6c2

Browse files
peterlimgpeterlimg
committed
Merge branch 'staging' into chain-fee
* staging: Modify/fmt leaf hash calculation (#912) Validator health (#994)
2 parents bd726c7 + fa0b557 commit b05f6c2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+2459
-1381
lines changed

‎code/go/0chain.net/blobbercore/allocation/common_test.go

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,10 @@ package allocation
22

33
import (
44
"bytes"
5-
"encoding/json"
65
"io"
76
"mime/multipart"
87

98
"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
10-
"github.com/0chain/gosdk/core/util"
119
)
1210

1311
type MockFileStore struct {
@@ -23,11 +21,11 @@ func (mfs *MockFileStore) WriteFile(allocID, connID string,
2321
b := bytes.NewBuffer(make([]byte, 0))
2422
n, _ := io.Copy(b, infile)
2523
return &filestore.FileOutputData{
26-
Name: fileData.Name,
27-
Path: fileData.Path,
28-
MerkleRoot: "",
29-
ContentHash: fileData.Hash,
30-
Size: n,
24+
Name: fileData.Name,
25+
Path: fileData.Path,
26+
FixedMerkleRoot: "",
27+
ValidationRoot: fileData.ValidationRoot,
28+
Size: n,
3129
}, nil
3230
}
3331

@@ -43,12 +41,12 @@ func (mfs *MockFileStore) DeleteFile(allocID string, contentHash string) error {
4341
return nil
4442
}
4543

46-
func (mfs *MockFileStore) GetFileBlock(allocID string, fileData *filestore.FileInputData, blockNum int64, numBlocks int64) ([]byte, error) {
44+
func (mfs *MockFileStore) GetFileBlock(rin *filestore.ReadBlockInput) (*filestore.FileDownloadResponse, error) {
4745
return nil, nil
4846
}
4947

50-
func (mfs *MockFileStore) GetBlocksMerkleTreeForChallenge(allocID string, fileData *filestore.FileInputData, blockoffset int) (json.RawMessage, util.MerkleTreeI, error) {
51-
return nil, nil, nil
48+
func (mfs *MockFileStore) GetBlocksMerkleTreeForChallenge(cir *filestore.ChallengeReadBlockInput) (*filestore.ChallengeResponse, error) {
49+
return nil, nil
5250
}
5351

5452
func (mfs *MockFileStore) GetTotalTempFileSizes() (s uint64) {

‎code/go/0chain.net/blobbercore/allocation/copyfilechange.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,10 @@ func (rf *CopyFileChange) ApplyChange(ctx context.Context, change *AllocationCha
111111

112112
func (rf *CopyFileChange) processCopyRefs(
113113
ctx context.Context, srcRef, destRef *reference.Ref,
114-
allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (
115-
fileRefs []*reference.Ref, err error) {
114+
allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string,
115+
) (
116+
fileRefs []*reference.Ref, err error,
117+
) {
116118

117119
newRef := *srcRef
118120
newRef.ID = 0

‎code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go

Lines changed: 53 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -71,70 +71,70 @@ func TestBlobberCore_CopyFile(t *testing.T) {
7171
mocket.Catcher.NewMock().WithQuery(query).WithReply(
7272
[]map[string]interface{}{
7373
{
74-
"id": 1,
75-
"level": 0,
76-
"lookup_hash": "lookup_hash_root",
77-
"path": "/",
78-
"name": "/",
79-
"allocation_id": alloc.ID,
80-
"parent_path": "",
81-
"content_hash": "",
82-
"thumbnail_size": 00,
83-
"thumbnail_hash": "",
84-
"type": reference.DIRECTORY,
85-
"created_at": common.Now() - 3600,
86-
"updated_at": common.Now() - 1800,
74+
"id": 1,
75+
"level": 0,
76+
"lookup_hash": "lookup_hash_root",
77+
"path": "/",
78+
"name": "/",
79+
"allocation_id": alloc.ID,
80+
"parent_path": "",
81+
"validation_root": "",
82+
"thumbnail_size": 00,
83+
"thumbnail_hash": "",
84+
"type": reference.DIRECTORY,
85+
"created_at": common.Now() - 3600,
86+
"updated_at": common.Now() - 1800,
8787
},
8888
{
89-
"id": 2,
90-
"level": 1,
91-
"lookup_hash": "lookup_hash",
92-
"path": "/orig.txt",
93-
"name": "orig.txt",
94-
"allocation_id": alloc.ID,
95-
"parent_path": "/",
96-
"content_hash": "content_hash",
97-
"thumbnail_size": 00,
98-
"thumbnail_hash": "",
99-
"type": reference.FILE,
100-
"created_at": common.Now() - 3600,
101-
"updated_at": common.Now() - 1800,
89+
"id": 2,
90+
"level": 1,
91+
"lookup_hash": "lookup_hash",
92+
"path": "/orig.txt",
93+
"name": "orig.txt",
94+
"allocation_id": alloc.ID,
95+
"parent_path": "/",
96+
"validation_root": "validation_root",
97+
"thumbnail_size": 00,
98+
"thumbnail_hash": "",
99+
"type": reference.FILE,
100+
"created_at": common.Now() - 3600,
101+
"updated_at": common.Now() - 1800,
102102
},
103103
},
104104
)
105105

106-
q2 := `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","file_meta_hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","chunk_size","lookup_hash","thumbnail_hash","allocation_root","level","created_at","updated_at" FROM "reference_objects" WHERE ((allocation_id=$1 AND parent_path=$2) OR (parent_path = $3 AND allocation_id = $4)) AND "reference_objects"."deleted_at" IS NULL ORDER BY path`
106+
q2 := `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","file_meta_hash","path_hash","validation_root","fixed_merkle_root","actual_file_size","actual_file_hash","chunk_size","lookup_hash","thumbnail_hash","allocation_root","level","created_at","updated_at" FROM "reference_objects" WHERE ((allocation_id=$1 AND parent_path=$2) OR (parent_path = $3 AND allocation_id = $4)) AND "reference_objects"."deleted_at" IS NULL ORDER BY path`
107107
mocket.Catcher.NewMock().WithQuery(q2).WithReply(
108108
[]map[string]interface{}{
109109
{
110-
"id": 1,
111-
"level": 1,
112-
"lookup_hash": "lookup_hash_root",
113-
"path": "/",
114-
"name": "/",
115-
"allocation_id": alloc.ID,
116-
"parent_path": "",
117-
"content_hash": "",
118-
"thumbnail_size": 00,
119-
"thumbnail_hash": "",
120-
"type": reference.DIRECTORY,
121-
"created_at": common.Now() - 3600,
122-
"updated_at": common.Now() - 1800,
110+
"id": 1,
111+
"level": 1,
112+
"lookup_hash": "lookup_hash_root",
113+
"path": "/",
114+
"name": "/",
115+
"allocation_id": alloc.ID,
116+
"parent_path": "",
117+
"validation_root": "",
118+
"thumbnail_size": 00,
119+
"thumbnail_hash": "",
120+
"type": reference.DIRECTORY,
121+
"created_at": common.Now() - 3600,
122+
"updated_at": common.Now() - 1800,
123123
},
124124
{
125-
"id": 2,
126-
"level": 2,
127-
"lookup_hash": "lookup_hash",
128-
"path": "/orig.txt",
129-
"name": "orig.txt",
130-
"allocation_id": alloc.ID,
131-
"parent_path": "/",
132-
"content_hash": "content_hash",
133-
"thumbnail_size": 00,
134-
"thumbnail_hash": "",
135-
"type": reference.FILE,
136-
"created_at": common.Now() - 3600,
137-
"updated_at": common.Now() - 1800,
125+
"id": 2,
126+
"level": 2,
127+
"lookup_hash": "lookup_hash",
128+
"path": "/orig.txt",
129+
"name": "orig.txt",
130+
"allocation_id": alloc.ID,
131+
"parent_path": "/",
132+
"validation_root": "validation_root",
133+
"thumbnail_size": 00,
134+
"thumbnail_hash": "",
135+
"type": reference.FILE,
136+
"created_at": common.Now() - 3600,
137+
"updated_at": common.Now() - 1800,
138138
},
139139
},
140140
)

‎code/go/0chain.net/blobbercore/allocation/deletefilechange.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,24 +65,24 @@ func (nf *DeleteFileChange) DeleteTempFile() error {
6565
func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error {
6666
db := datastore.GetStore().GetTransaction(ctx)
6767
type Result struct {
68-
Id string
69-
ContentHash string
70-
ThumbnailHash string
68+
Id string
69+
ValidationRoot string
70+
ThumbnailHash string
7171
}
7272

7373
limitCh := make(chan struct{}, 10)
7474
wg := &sync.WaitGroup{}
7575
var results []Result
7676
err := db.Model(&reference.Ref{}).Unscoped().
77-
Select("id", "content_hash", "thumbnail_hash").
77+
Select("id", "validation_root", "thumbnail_hash").
7878
Where("allocation_id=? AND path LIKE ? AND type=? AND deleted_at is not NULL",
7979
nf.AllocationID, nf.Path+"%", reference.FILE).
8080
FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error {
8181

8282
for _, res := range results {
8383
var count int64
8484
tx.Model(&reference.Ref{}).
85-
Where("allocation_id=? AND content_hash=?", nf.AllocationID, res.ContentHash).
85+
Where("allocation_id=? AND validation_root=?", nf.AllocationID, res.ValidationRoot).
8686
Count(&count)
8787

8888
db.Model(&reference.CommitMetaTxn{}).Delete("ref_id = ?", res.Id)
@@ -101,10 +101,10 @@ func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error {
101101
}()
102102

103103
if count == 0 {
104-
err := filestore.GetFileStore().DeleteFile(nf.AllocationID, res.ContentHash)
104+
err := filestore.GetFileStore().DeleteFile(nf.AllocationID, res.ValidationRoot)
105105
if err != nil {
106106
logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()),
107-
zap.String("content_hash", res.ContentHash))
107+
zap.String("validation_root", res.ValidationRoot))
108108
}
109109
}
110110

‎code/go/0chain.net/blobbercore/allocation/file_changer_base.go

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ type BaseFileChanger struct {
1717
//client side:
1818
Path string `json:"filepath" validation:"required"`
1919
//client side:
20+
ActualFileHashSignature string `json:"actual_file_hash_signature,omitempty" validation:"required"`
21+
//client side:
2022
ActualHash string `json:"actual_hash,omitempty" validation:"required"`
2123
//client side:
2224
ActualSize int64 `json:"actual_size,omitempty" validation:"required"`
@@ -29,13 +31,15 @@ type BaseFileChanger struct {
2931
MimeType string `json:"mimetype,omitempty"`
3032
//client side:
3133
//client side:
32-
MerkleRoot string `json:"merkle_root,omitempty"`
34+
FixedMerkleRoot string `json:"fixed_merkle_root,omitempty"`
3335

3436
//server side: update them by ChangeProcessor
3537
AllocationID string `json:"allocation_id"`
3638
//client side:
37-
Hash string `json:"content_hash,omitempty"`
38-
Size int64 `json:"size"`
39+
ValidationRootSignature string `json:"validation_root_signature,omitempty"`
40+
//client side:
41+
ValidationRoot string `json:"validation_root,omitempty"`
42+
Size int64 `json:"size"`
3943
//server side:
4044
ThumbnailHash string `json:"thumbnail_content_hash,omitempty"`
4145
ThumbnailSize int64 `json:"thumbnail_size"`
@@ -57,13 +61,13 @@ func (fc *BaseFileChanger) DeleteTempFile() error {
5761
fileInputData := &filestore.FileInputData{}
5862
fileInputData.Name = fc.Filename
5963
fileInputData.Path = fc.Path
60-
fileInputData.Hash = fc.Hash
64+
fileInputData.ValidationRoot = fc.ValidationRoot
6165
err := filestore.GetFileStore().DeleteTempFile(fc.AllocationID, fc.ConnectionID, fileInputData)
6266
if fc.ThumbnailSize > 0 {
6367
fileInputData := &filestore.FileInputData{}
6468
fileInputData.Name = fc.ThumbnailFilename
6569
fileInputData.Path = fc.Path
66-
fileInputData.Hash = fc.ThumbnailHash
70+
fileInputData.ThumbnailHash = fc.ThumbnailHash
6771
err = filestore.GetFileStore().DeleteTempFile(fc.AllocationID, fc.ConnectionID, fileInputData)
6872
}
6973
return err
@@ -73,8 +77,8 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context) error {
7377
fileInputData := &filestore.FileInputData{}
7478
fileInputData.Name = fc.Filename
7579
fileInputData.Path = fc.Path
76-
fileInputData.Hash = fc.Hash
77-
fileInputData.MerkleRoot = fc.MerkleRoot
80+
fileInputData.ValidationRoot = fc.ValidationRoot
81+
fileInputData.FixedMerkleRoot = fc.FixedMerkleRoot
7882
fileInputData.ChunkSize = fc.ChunkSize
7983
_, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData)
8084
if err != nil {
@@ -84,7 +88,7 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context) error {
8488
fileInputData := &filestore.FileInputData{}
8589
fileInputData.Name = fc.ThumbnailFilename
8690
fileInputData.Path = fc.Path
87-
fileInputData.Hash = fc.ThumbnailHash
91+
fileInputData.ThumbnailHash = fc.ThumbnailHash
8892
fileInputData.ChunkSize = fc.ChunkSize
8993
fileInputData.IsThumbnail = true
9094
_, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData)

‎code/go/0chain.net/blobbercore/allocation/file_changer_update.go

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -78,15 +78,17 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, change *Allocation
7878
if fileRef.ThumbnailHash != "" && fileRef.ThumbnailHash != nf.ThumbnailHash {
7979
nf.deleteHash[fileRef.ThumbnailHash] = true
8080
}
81-
if fileRef.ContentHash != "" && fileRef.ContentHash != nf.Hash {
82-
nf.deleteHash[fileRef.ContentHash] = true
81+
if fileRef.ValidationRoot != "" && fileRef.ValidationRoot != nf.ValidationRoot {
82+
nf.deleteHash[fileRef.ValidationRoot] = true
8383
}
8484
fileRef.ActualFileHash = nf.ActualHash
85+
fileRef.ActualFileHashSignature = nf.ActualFileHashSignature
8586
fileRef.ActualFileSize = nf.ActualSize
8687
fileRef.MimeType = nf.MimeType
87-
fileRef.ContentHash = nf.Hash
88+
fileRef.ValidationRootSignature = nf.ValidationRootSignature
89+
fileRef.ValidationRoot = nf.ValidationRoot
8890
fileRef.CustomMeta = nf.CustomMeta
89-
fileRef.MerkleRoot = nf.MerkleRoot
91+
fileRef.FixedMerkleRoot = nf.FixedMerkleRoot
9092
fileRef.AllocationRoot = allocationRoot
9193
fileRef.Size = nf.Size
9294
fileRef.ThumbnailHash = nf.ThumbnailHash
@@ -108,18 +110,18 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, change *Allocation
108110

109111
func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context) error {
110112
db := datastore.GetStore().GetTransaction(ctx)
111-
for contenthash := range nf.deleteHash {
113+
for hash := range nf.deleteHash {
112114
var count int64
113115
err := db.Table((&reference.Ref{}).TableName()).
114116
Where(
115-
db.Where(&reference.Ref{ThumbnailHash: contenthash}).
116-
Or(&reference.Ref{ContentHash: contenthash})).
117+
db.Where(&reference.Ref{ThumbnailHash: hash}).
118+
Or(&reference.Ref{ValidationRoot: hash})).
117119
Where(&reference.Ref{AllocationID: nf.AllocationID}).
118120
Count(&count).Error
119121

120122
if err == nil && count == 0 {
121-
logging.Logger.Info("Deleting content file", zap.String("content_hash", contenthash))
122-
if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil {
123+
logging.Logger.Info("Deleting content file", zap.String("validation_root", hash))
124+
if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, hash); err != nil {
123125
logging.Logger.Error("FileStore_DeleteFile", zap.String("allocation_id", nf.AllocationID), zap.Error(err))
124126
}
125127
}

‎code/go/0chain.net/blobbercore/allocation/file_changer_upload.go

Lines changed: 24 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -84,28 +84,30 @@ func (nf *UploadFileChanger) ApplyChange(ctx context.Context, change *Allocation
8484
}
8585

8686
newFile := &reference.Ref{
87-
ActualFileHash: nf.ActualHash,
88-
ActualFileSize: nf.ActualSize,
89-
AllocationID: dirRef.AllocationID,
90-
ContentHash: nf.Hash,
91-
CustomMeta: nf.CustomMeta,
92-
MerkleRoot: nf.MerkleRoot,
93-
Name: nf.Filename,
94-
Path: nf.Path,
95-
ParentPath: dirRef.Path,
96-
Type: reference.FILE,
97-
Size: nf.Size,
98-
MimeType: nf.MimeType,
99-
AllocationRoot: allocationRoot,
100-
ThumbnailHash: nf.ThumbnailHash,
101-
ThumbnailSize: nf.ThumbnailSize,
102-
ActualThumbnailHash: nf.ActualThumbnailHash,
103-
ActualThumbnailSize: nf.ActualThumbnailSize,
104-
EncryptedKey: nf.EncryptedKey,
105-
ChunkSize: nf.ChunkSize,
106-
CreatedAt: ts,
107-
UpdatedAt: ts,
108-
HashToBeComputed: true,
87+
ActualFileHash: nf.ActualHash,
88+
ActualFileHashSignature: nf.ActualFileHashSignature,
89+
ActualFileSize: nf.ActualSize,
90+
AllocationID: dirRef.AllocationID,
91+
ValidationRoot: nf.ValidationRoot,
92+
ValidationRootSignature: nf.ValidationRootSignature,
93+
CustomMeta: nf.CustomMeta,
94+
FixedMerkleRoot: nf.FixedMerkleRoot,
95+
Name: nf.Filename,
96+
Path: nf.Path,
97+
ParentPath: dirRef.Path,
98+
Type: reference.FILE,
99+
Size: nf.Size,
100+
MimeType: nf.MimeType,
101+
AllocationRoot: allocationRoot,
102+
ThumbnailHash: nf.ThumbnailHash,
103+
ThumbnailSize: nf.ThumbnailSize,
104+
ActualThumbnailHash: nf.ActualThumbnailHash,
105+
ActualThumbnailSize: nf.ActualThumbnailSize,
106+
EncryptedKey: nf.EncryptedKey,
107+
ChunkSize: nf.ChunkSize,
108+
CreatedAt: ts,
109+
UpdatedAt: ts,
110+
HashToBeComputed: true,
109111
}
110112

111113
fileID, ok := fileIDMeta[newFile.Path]

0 commit comments

Comments
 (0)