Skip to content

Commit

Permalink
Merge pull request #26 from bnb-chain/develop
Browse files Browse the repository at this point in the history 
release: prepare for release v0.0.6
  • Loading branch information
@flywukong
flywukong authored May 12, 2023
2 parents ed19e36 + 42e346d commit 33b0f12
Show file tree
Hide file tree
Showing 5 changed files with 269 additions and 33 deletions.
10 changes: 10 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,16 @@ func ComputeIntegrityHash(reader io.Reader, segmentSize int64, dataShards, parit

// compute hash roots based on file path
func ComputerHashFromFile(filePath string, segmentSize int64, dataShards, parityShards int) ([]string, int64, error)

// IntegrityHasher is used to calculate integrityHash in a stream way. It contains Init, Append, and Finish functions.
IntegrityHasher := NewHasher(segmentSize, dataShards, parityShards)
IntegrityHasher.Init()

// append the data chunks to IntegrityHasher, the data size should be less than segment size
func (i *IntegrityHasher) Append(data []byte) error

// compute the result of the Integrity hashes
func (i *IntegrityHasher) Finish() ([][]byte, int64, storageTypes.RedundancyType, error) {
```
### 3. Generate checksum and integrity hash
Expand Down
17 changes: 9 additions & 8 deletions go/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/bnb-chain/greenfield-common/go
go 1.18

require (
github.com/bnb-chain/greenfield v0.0.10
github.com/bnb-chain/greenfield v0.1.2
github.com/ethereum/go-ethereum v1.10.19
github.com/klauspost/reedsolomon v1.11.6
github.com/rs/zerolog v1.29.0
Expand Down Expand Up @@ -119,13 +119,13 @@ require (
go.etcd.io/bbolt v1.3.6 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/term v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/term v0.7.0 // indirect
golang.org/x/text v0.9.0 // indirect
google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 // indirect
google.golang.org/grpc v1.53.0 // indirect
google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 // indirect
google.golang.org/grpc v1.54.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand All @@ -135,6 +135,7 @@ require (
replace (
cosmossdk.io/math => github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20230228075616-68ac309b432c
github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0
github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v0.0.11
github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v0.1.1
github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
github.com/tendermint/tendermint => github.com/bnb-chain/greenfield-tendermint v0.0.4
)
36 changes: 18 additions & 18 deletions go/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,12 +146,14 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
github.com/bnb-chain/greenfield v0.0.10 h1:WACUvR977jK4jDm0LTcYl7gomXtn5mUXXAPoGo9s+dI=
github.com/bnb-chain/greenfield v0.0.10/go.mod h1:UDWHU9XiyTbbpZsktn1bMab4MVAY4dhWEf/mhMuOQgQ=
github.com/bnb-chain/greenfield-cosmos-sdk v0.0.11 h1:8Vb57cJUoMVv9EL5ce43V8ZCPOvjNjrxs9+KyJw6JOg=
github.com/bnb-chain/greenfield-cosmos-sdk v0.0.11/go.mod h1:C0poOez5FBpH4Y/oa9jmco2fRojeMsJZGOZEX+GOv8w=
github.com/bnb-chain/greenfield v0.1.2 h1:9ZtfVDxa28DrhosTJIyxge3fYrafDW3XKd3hhphv7Q8=
github.com/bnb-chain/greenfield v0.1.2/go.mod h1:nNOm0RYPJRrgWpEZ+Wak107amfeykqit9adaL38VfWA=
github.com/bnb-chain/greenfield-cosmos-sdk v0.1.1 h1:7lH3Iu50PXE48a/tzt1SklEDRPP14LCToPzrAz57VUw=
github.com/bnb-chain/greenfield-cosmos-sdk v0.1.1/go.mod h1:qHW5MF6lhzlQuUg6pn7sYFSPZOO/AXr42ieRCApAW0k=
github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20230228075616-68ac309b432c h1:BLmdYaj7Dx0YOhfk77+KPPJSMCwpQl6f4Y30+801bf0=
github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20230228075616-68ac309b432c/go.mod h1:u/MXvf8wbUbCsAEyQSSYXXMsczAsFX48e2D6JI86T4o=
github.com/bnb-chain/greenfield-tendermint v0.0.4 h1:Dh3FYn/S8ja4+OE3uSqL3jdJ0flT6O2GlCkaQCXb4SI=
github.com/bnb-chain/greenfield-tendermint v0.0.4/go.mod h1:/v9z9F6cq0+f7EGG92lYSLBcPYQDILoK91X8YM28hWo=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=
Expand Down Expand Up @@ -1276,8 +1278,6 @@ github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15 h1:hqAk8riJvK4RM
github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15/go.mod h1:z4YtwM70uOnk8h0pjJYlj3zdYwi9l03By6iAIF5j/Pk=
github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2lyGa2E=
github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME=
github.com/tendermint/tendermint v0.34.22 h1:XMhtC8s8QqJO4l/dn+TkQvevTRSow3Vixjclr41o+2Q=
github.com/tendermint/tendermint v0.34.22/go.mod h1:YpP5vBEAKUT4g6oyfjKgFeZmdB/GjkJAxfF+cgmJg6Y=
github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8=
github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I=
github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e h1:cR8/SYRgyQCt5cNCMniB/ZScMkhI9nk8U5C7SbISXjo=
Expand Down Expand Up @@ -1526,8 +1526,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -1659,13 +1659,13 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand All @@ -1676,8 +1676,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down Expand Up @@ -1876,8 +1876,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc=
google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag=
google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
Expand All @@ -1892,8 +1892,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 h1:KR8+MyP7/qOlV+8Af01LtjL04bu7on42eVsxT4EyBQk=
google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610=
gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI=
Expand Down
148 changes: 142 additions & 6 deletions go/hash/hash.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package hash

import (
"bytes"
"errors"
"io"
"os"
"sync"
Expand All @@ -12,9 +13,144 @@ import (
"github.com/bnb-chain/greenfield-common/go/redundancy"
)

// IntegrityHasher compute integrityHash
type IntegrityHasher struct {
ecDataHashes [][][]byte
segHashes [][]byte
buffer []byte
segmentSize int64
dataShards int
parityShards int
contentLen int64
}

func NewHasher(size int64, data, parity int) *IntegrityHasher {
return &IntegrityHasher{
buffer: make([]byte, 0),
segmentSize: size,
dataShards: data,
parityShards: parity,
}
}

// Init the integrityHash fields
func (i *IntegrityHasher) Init() {
ecShards := i.dataShards + i.parityShards
encodeDataHash := make([][][]byte, ecShards)
for i := 0; i < ecShards; i++ {
encodeDataHash[i] = make([][]byte, 0)
}
segChecksumList := make([][]byte, 0)
i.ecDataHashes = encodeDataHash
i.segHashes = segChecksumList
if len(i.buffer) > 0 {
i.buffer = i.buffer[:0]
}
i.contentLen = 0
}

// Append the data chunks to IntegrityHasher , the data size should be less than segment size
func (i *IntegrityHasher) Append(data []byte) error {
dataSize := len(data)
if dataSize > int(i.segmentSize) {
return errors.New("the length of data size should be less than segmentSize")
}
if len(i.buffer) >= int(i.segmentSize) {
return errors.New("the buffer of handler should be less than segmentSize")
}
originBuffer := make([]byte, len(i.buffer))
copy(originBuffer, i.buffer)
// use tempBuffer to store exceed data
var tempBuffer []byte
totalSize := int64(dataSize + len(i.buffer))
if totalSize > i.segmentSize {
index := dataSize - int(totalSize-i.segmentSize)
tempBuffer = make([]byte, dataSize-index)
copy(tempBuffer, data[index:])
// buffer should be equal with segment size
i.buffer = append(i.buffer, data[:index]...)
} else {
i.buffer = append(i.buffer, data...)
// if buffer size less than segment size, just store the data
if totalSize < i.segmentSize {
return nil
}
}

// compute segment hash
if err := i.computeBufferHash(); err != nil {
return err
}

// copy exceed data to buffer if exist
if len(tempBuffer) > 0 {
i.buffer = i.buffer[:0]
i.buffer = append(i.buffer, tempBuffer...)
return nil
}

i.buffer = i.buffer[:0]
return nil
}

// Finish return the result of the Integrity hashes
func (i *IntegrityHasher) Finish() ([][]byte, int64, storageTypes.RedundancyType, error) {
// deal with remain content tot be computed
if len(i.buffer) > 0 {
if err := i.computeBufferHash(); err != nil {
return nil, 0, storageTypes.REDUNDANCY_EC_TYPE, err
}
}

hashList := make([][]byte, i.parityShards+i.dataShards+1)

hashList[0] = GenerateIntegrityHash(i.segHashes)

wg := &sync.WaitGroup{}
spLen := len(i.ecDataHashes)
wg.Add(spLen)
for spID, content := range i.ecDataHashes {
go func(data [][]byte, id int) {
defer wg.Done()
hashList[id+1] = GenerateIntegrityHash(data)
}(content, spID)
}
wg.Wait()

return hashList, i.contentLen, storageTypes.REDUNDANCY_EC_TYPE, nil
}

// computeBufferHash erasure encode the buffer of IntegrityHasher and compute the hash
func (i *IntegrityHasher) computeBufferHash() error {
i.contentLen += int64(len(i.buffer))
originBuffer := make([]byte, len(i.buffer))
copy(originBuffer, i.buffer)
// compute segment hash
checksum := GenerateChecksum(i.buffer)
i.segHashes = append(i.segHashes, checksum)
// get erasure encoded bytes and compute pieces hashes
encodeShards, err := redundancy.EncodeRawSegment(i.buffer, i.dataShards, i.parityShards)
if err != nil {
// recover buffer content if encode error
i.buffer = i.buffer[:0]
i.buffer = append(i.buffer, originBuffer...)
return err
}

for index, shard := range encodeShards {
// compute hash of pieces
piecesHash := GenerateChecksum(shard)
i.ecDataHashes[index] = append(i.ecDataHashes[index], piecesHash)
}

return nil
}

// ComputeIntegrityHash split the reader into segment, ec encode the data, compute the hash roots of pieces
// return the hash result array list and data size
func ComputeIntegrityHash(reader io.Reader, segmentSize int64, dataShards, parityShards int) ([][]byte, int64, storageTypes.RedundancyType, error) {
// return the hash result array list and data segmentSize
func ComputeIntegrityHash(reader io.Reader, segmentSize int64, dataShards, parityShards int) ([][]byte, int64,
storageTypes.RedundancyType, error,
) {
var segChecksumList [][]byte
ecShards := dataShards + parityShards

Expand All @@ -25,7 +161,7 @@ func ComputeIntegrityHash(reader io.Reader, segmentSize int64, dataShards, parit

hashList := make([][]byte, ecShards+1)
contentLen := int64(0)
// read the data by segment size
// read the data by segment segmentSize
for {
seg := make([]byte, segmentSize)
n, err := reader.Read(seg)
Expand Down Expand Up @@ -60,7 +196,7 @@ func ComputeIntegrityHash(reader io.Reader, segmentSize int64, dataShards, parit
// combine the hash root of pieces of the PrimarySP
hashList[0] = GenerateIntegrityHash(segChecksumList)

// compute the hash root of pieces of the SecondarySP
// compute the integrity hash of the SecondarySP
wg := &sync.WaitGroup{}
spLen := len(encodeDataHash)
wg.Add(spLen)
Expand All @@ -76,7 +212,7 @@ func ComputeIntegrityHash(reader io.Reader, segmentSize int64, dataShards, parit
return hashList, contentLen, storageTypes.REDUNDANCY_EC_TYPE, nil
}

// ComputerHashFromFile open a local file and compute hash result and size
// ComputerHashFromFile open a local file and compute hash result and segmentSize
func ComputerHashFromFile(filePath string, segmentSize int64, dataShards, parityShards int) ([][]byte, int64, storageTypes.RedundancyType, error) {
f, err := os.Open(filePath)
if err != nil {
Expand All @@ -88,7 +224,7 @@ func ComputerHashFromFile(filePath string, segmentSize int64, dataShards, parity
return ComputeIntegrityHash(f, segmentSize, dataShards, parityShards)
}

// ComputerHashFromBuffer support computing hash and size from byte buffer
// ComputerHashFromBuffer support computing hash and segmentSize from byte buffer
func ComputerHashFromBuffer(content []byte, segmentSize int64, dataShards, parityShards int) ([][]byte, int64, storageTypes.RedundancyType, error) {
reader := bytes.NewReader(content)
return ComputeIntegrityHash(reader, segmentSize, dataShards, parityShards)
Expand Down
Loading

0 comments on commit 33b0f12

Please sign in to comment.