Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix mem usage #1337

Merged
merged 5 commits into from
Dec 2, 2023
Merged

Fix mem usage #1337

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
100e569
updated gosdk to blobber
shahnawaz-creator Dec 1, 2023
e28b7c0
change hasher to sha2 256
Hitenjain14 Dec 1, 2023
1689326
merge
Hitenjain14 Dec 1, 2023
3a258af
fix unit test
Hitenjain14 Dec 1, 2023
c6914f5
changed runner for lint tests.yml
shahnawaz-creator Dec 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion code/go/0chain.net/blobbercore/filestore/store_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -801,7 +801,7 @@ func TestGetMerkleTree(t *testing.T) {

rootHash, _ := hex.DecodeString(fixedMerkleRoot)
fmp := &util.FixedMerklePath{
LeafHash: encryption.BlakeHash(challengeProof.Data),
LeafHash: encryption.ShaHash(challengeProof.Data),
RootHash: rootHash,
Nodes: challengeProof.Proof,
LeafInd: test.blockOffset,
Expand Down
6 changes: 3 additions & 3 deletions code/go/0chain.net/blobbercore/filestore/tree_validation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"sync"

"github.com/0chain/gosdk/core/util"
"github.com/zeebo/blake3"
"github.com/minio/sha256-simd"
)

const (
Expand Down Expand Up @@ -66,7 +66,7 @@ func (ft *fixedMerkleTree) CalculateRootAndStoreNodes(f io.Writer) (merkleRoot [

buffer := make([]byte, FMTSize)
var bufLen int
h := blake3.New()
h := sha256.New()

for i := 0; i < util.FixedMTDepth; i++ {
if len(nodes) == 1 {
Expand Down Expand Up @@ -228,7 +228,7 @@ func (v *validationTree) CalculateRootAndStoreNodes(f io.WriteSeeker) (merkleRoo
nodes := make([][]byte, len(v.GetLeaves()))
copy(nodes, v.GetLeaves())

h := blake3.New()
h := sha256.New()
depth := v.CalculateDepth()

s := getNodesSize(v.GetDataSize(), util.MaxMerkleLeavesSize)
Expand Down
4 changes: 2 additions & 2 deletions code/go/0chain.net/blobbercore/filestore/tree_validation_bench_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"testing"

"github.com/0chain/gosdk/core/util"
"github.com/zeebo/blake3"
"github.com/minio/sha256-simd"
)

func BenchmarkFixedMerkleProofFor10MB(b *testing.B) {
Expand Down Expand Up @@ -81,7 +81,7 @@ func runFixedMPBench(b *testing.B, size int64, filename string) {
b.Fatalf("error: %v", err)
}

h := blake3.New()
h := sha256.New()
_, _ = h.Write(proofByte)

fp := util.FixedMerklePath{
Expand Down
4 changes: 2 additions & 2 deletions code/go/0chain.net/blobbercore/filestore/tree_validation_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ func TestFixedMerkleTreeProof(t *testing.T) {
proofByte, err := fm.GetLeafContent(r)
require.NoError(t, err)

leafHash := encryption.BlakeHash(proofByte)
leafHash := encryption.ShaHash(proofByte)
ftp := util.FixedMerklePath{
LeafHash: leafHash,
RootHash: merkleRoot,
Expand Down Expand Up @@ -231,7 +231,7 @@ func TestValidationTreeWrite(t *testing.T) {
n, err = r.Read(b)
require.NoError(t, err)
require.EqualValues(t, size, n)
leaves[0] = encryption.BlakeHash(b)
leaves[0] = encryption.ShaHash(b)
} else {
nodes := make([]byte, totalLeaves*HashSize)
n, err = r.Read(nodes)
Expand Down
6 changes: 4 additions & 2 deletions code/go/0chain.net/blobbercore/handler/client_quota.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,10 @@ func saveClientStats() {
}
mpLock.Unlock()
_ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
tx := datastore.GetStore().GetTransaction(ctx)
tx.Create(&dbStats)
if len(dbStats) > 0 {
tx := datastore.GetStore().GetTransaction(ctx)
return tx.Create(dbStats).Error
}
return nil
})
var blackList []string
Expand Down
6 changes: 3 additions & 3 deletions code/go/0chain.net/core/encryption/hash.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"crypto/sha1"
"encoding/hex"

"github.com/zeebo/blake3"
"github.com/minio/sha256-simd"
"golang.org/x/crypto/sha3"
)

Expand Down Expand Up @@ -35,7 +35,7 @@ func RawHash(data interface{}) []byte {
return hash.Sum(nil)
}

func BlakeHash(data interface{}) []byte {
func ShaHash(data interface{}) []byte {
var databuf []byte
switch dataImpl := data.(type) {
case []byte:
Expand All @@ -47,7 +47,7 @@ func BlakeHash(data interface{}) []byte {
default:
panic("unknown type")
}
hash := blake3.New()
hash := sha256.New()
_, _ = hash.Write(databuf)
return hash.Sum(nil)
}
Expand Down
2 changes: 1 addition & 1 deletion code/go/0chain.net/validatorcore/storage/models.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ func (cr *ChallengeRequest) VerifyChallenge(challengeObj *Challenge, allocationO
}

logging.Logger.Info("Verifying data block and merkle path", zap.String("challenge_id", challengeObj.ID))
fHash := encryption.BlakeHash(cr.ChallengeProof.Data)
fHash := encryption.ShaHash(cr.ChallengeProof.Data)
fixedMerkleRoot, _ := hex.DecodeString(cr.ObjPath.Meta.FixedMerkleRoot)
fmp := &util.FixedMerklePath{
LeafHash: fHash,
Expand Down
7 changes: 3 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.18

require (
github.com/0chain/errors v1.0.3
github.com/0chain/gosdk v1.10.1-0.20231114143641-51b9f0240b64
github.com/0chain/gosdk v1.10.1-0.20231201213614-42fc90052217
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/didip/tollbooth/v6 v6.1.2
github.com/go-openapi/runtime v0.26.0
Expand Down Expand Up @@ -44,8 +44,7 @@ require (

require (
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/zeebo/blake3 v0.2.3
github.com/minio/sha256-simd v1.0.1
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
)

Expand All @@ -67,7 +66,7 @@ require (
github.com/emirpasic/gods v1.18.1
github.com/ethereum/go-ethereum v1.11.4 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/fsnotify/fsnotify v1.6.0
github.com/gin-gonic/gin v1.9.1 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-openapi/analysis v0.21.4 // indirect
Expand Down
11 changes: 2 additions & 9 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs
github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E=
github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM=
github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc=
github.com/0chain/gosdk v1.10.1-0.20231114143641-51b9f0240b64 h1:DBDslN917WflFB3FFWBU9ISgym5tye0gBVw0N3m7Fmk=
github.com/0chain/gosdk v1.10.1-0.20231114143641-51b9f0240b64/go.mod h1:l/c+X5v6RMyzWUib6FFPdK5bTAzAJ7PojhEvELp3LGk=
github.com/0chain/gosdk v1.10.1-0.20231201213614-42fc90052217 h1:ZkNwTdRdBaTFbfQ3POs+KhzEK98zDMb+b9OstNigwDs=
github.com/0chain/gosdk v1.10.1-0.20231201213614-42fc90052217/go.mod h1:DAg/de6vodjEa7CM1/LjElOwntRtNV5lb9rMRaR7fzU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
Expand Down Expand Up @@ -479,7 +479,6 @@ github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY=
Expand Down Expand Up @@ -745,12 +744,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs=
go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw=
go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ=
Expand Down
Loading