Refactor registry system: no direct dependencies; expose standard hash.Hash; be a data carrier.

.travis.yml

@@ -4,7 +4,7 @@ os:
 language: go
 
 go:
-  - 1.11.x
+  - 1.15.x
 
 env:
   global:

core/errata.go

@@ -0,0 +1,34 @@
+package multihash
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"hash"
+)
+
+type identityMultihash struct {
+	bytes.Buffer
+}
+
+func (identityMultihash) BlockSize() int {
+	return 32 // A prefered block size is nonsense for the "identity" "hash".  An arbitrary but unsurprising and positive nonzero number has been chosen to minimize the odds of fascinating bugs.
+}
+
+func (x *identityMultihash) Size() int {
+	return x.Len()
+}
+
+func (x *identityMultihash) Sum(digest []byte) []byte {
+	return x.Bytes()
+}
+
+type doubleSha256 struct {
+	hash.Hash
+}
+
+func (x doubleSha256) Sum(digest []byte) []byte {
+	digest = x.Hash.Sum(digest)
+	h2 := sha256.New()
+	h2.Write(digest)
+	return h2.Sum(digest[0:0])
+}

core/magic.go

@@ -0,0 +1,26 @@
+package multihash
+
+import "errors"
+
+// ErrSumNotSupported is returned when the Sum function code is not implemented
+var ErrSumNotSupported = errors.New("no such hash registered")
+
+// constants
+const (
+	IDENTITY     = 0x00
+	SHA1         = 0x11
+	SHA2_256     = 0x12
+	SHA2_512     = 0x13
+	SHA3_224     = 0x17
+	SHA3_256     = 0x16
+	SHA3_384     = 0x15
+	SHA3_512     = 0x14
+	KECCAK_224   = 0x1A
+	KECCAK_256   = 0x1B
+	KECCAK_384   = 0x1C
+	KECCAK_512   = 0x1D
+	SHAKE_128    = 0x18
+	SHAKE_256    = 0x19
+	MD5          = 0xd5
+	DBL_SHA2_256 = 0x56
+)

core/registry.go

@@ -0,0 +1,77 @@
+package multihash
+
+import (
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"fmt"
+	"hash"
+)
+
+// registry is a simple map which maps a multihash indicator number
+// to a standard golang Hash interface.
+//
+// Multihash indicator numbers are reserved and described in
+// https://github.com/multiformats/multicodec/blob/master/table.csv .
+// The keys used in this map must match those reservations.
+//
+// Hashers which are available in the golang stdlib will be registered automatically.
+// Others can be added using the Register function.
+var registry = make(map[uint64]func() hash.Hash)
+
+// Register adds a new hash to the set available from GetHasher and Sum.
+//
+// Register has a global effect and should only be used at package init time to avoid data races.
+//
+// The indicator code should be per the numbers reserved and described in
+// https://github.com/multiformats/multicodec/blob/master/table.csv .
+//
+// If Register is called with the same indicator code more than once, the last call wins.
+// In practice, this means that if an application has a strong opinion about what implementation to use for a certain hash
+// (e.g., perhaps they want to override the sha256 implementation to use a special hand-rolled assembly variant
+// rather than the stdlib one which is registered by default),
+// then this can be done by making a Register call with that effect at init time in the application's main package.
+// This should have the desired effect because the root of the import tree has its init time effect last.
+func Register(indicator uint64, hasherFactory func() hash.Hash) {
+	if hasherFactory == nil {
+		panic("not sensible to attempt to register a nil function")
+	}
+	registry[indicator] = hasherFactory
+	DefaultLengths[indicator] = hasherFactory().Size()
+}
+
+// GetHasher returns a new hash.Hash according to the indicator code number provided.
+//
+// The indicator code should be per the numbers reserved and described in
+// https://github.com/multiformats/multicodec/blob/master/table.csv .
+//
+// The actual hashers available are determined by what has been registered.
+// The registry automatically contains those hashers which are available in the golang standard libraries
+// (which includes md5, sha1, sha256, sha384, sha512, and the "identity" mulithash, among others).
+// Other hash implementations can be made available by using the Register function.
+// The 'go-mulithash/register/*' packages can also be imported to gain more common hash functions.
+//
+// If an error is returned, it will match `errors.Is(err, ErrSumNotSupported)`.
+func GetHasher(indicator uint64) (hash.Hash, error) {
+	factory, exists := registry[indicator]
+	if !exists {
+		return nil, fmt.Errorf("unknown multihash code %d (0x%x): %w", indicator, indicator, ErrSumNotSupported)
+	}
+	return factory(), nil
+}
+
+// DefaultLengths maps a multihash indicator code to the output size for that hash, in units of bytes.
+//
+// This map is populated when a hash function is registered by the Register function.
+// It's effectively a shortcut for asking Size() on the hash.Hash.
+var DefaultLengths = map[uint64]int{}
+
+func init() {
+	Register(IDENTITY, func() hash.Hash { return &identityMultihash{} })
+	Register(MD5, md5.New)
+	Register(SHA1, sha1.New)
+	Register(SHA2_256, sha256.New)
+	Register(SHA2_512, sha512.New)
+	Register(DBL_SHA2_256, func() hash.Hash { return &doubleSha256{sha256.New()} })
+}

go.mod

@@ -2,11 +2,11 @@ module github.com/multiformats/go-multihash
 
 require (
 	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
-	github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771
-	github.com/mr-tron/base58 v1.1.3
-	github.com/multiformats/go-varint v0.0.5
-	github.com/spaolacci/murmur3 v1.1.0
-	golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8
+	github.com/minio/sha256-simd v1.0.0
+	github.com/mr-tron/base58 v1.2.0
+	github.com/multiformats/go-varint v0.0.6
+	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
+	golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect
 )
 
 go 1.13

go.sum

@@ -1,22 +1,20 @@
+github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
-github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
-github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
-github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
-github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
-github.com/multiformats/go-varint v0.0.3 h1:1OZFaq4XbSNQE6ujqgr6/EIZlgHE7DmojAFsLqAJ26M=
-github.com/multiformats/go-varint v0.0.3/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
-github.com/multiformats/go-varint v0.0.4 h1:CplQWhUouUgTZ53vNFE8VoWr2VjaKXci+xyrKyyFuSw=
-github.com/multiformats/go-varint v0.0.4/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
-github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
-github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
-github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
-github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
+github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
+github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
+github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

multihash.go

@@ -80,7 +80,6 @@ func init() {
 		name := fmt.Sprintf("blake2b-%d", n*8)
 		Names[name] = c
 		Codes[c] = name
-		DefaultLengths[c] = int(n)
 	}
 
 	// Add blake2s (32 codes)
@@ -89,7 +88,6 @@ func init() {
 		name := fmt.Sprintf("blake2s-%d", n*8)
 		Names[name] = c
 		Codes[c] = name
-		DefaultLengths[c] = int(n)
 	}
 }
 
@@ -142,28 +140,6 @@ var Codes = map[uint64]string{
 	MD5:                       "md5",
 }
 
-// DefaultLengths maps a hash code to it's default length
-var DefaultLengths = map[uint64]int{
-	IDENTITY:     -1,
-	SHA1:         20,
-	SHA2_256:     32,
-	SHA2_512:     64,
-	SHA3_224:     28,
-	SHA3_256:     32,
-	SHA3_384:     48,
-	SHA3_512:     64,
-	DBL_SHA2_256: 32,
-	KECCAK_224:   28,
-	KECCAK_256:   32,
-	MURMUR3_128:  4,
-	KECCAK_384:   48,
-	KECCAK_512:   64,
-	SHAKE_128:    32,
-	SHAKE_256:    64,
-	X11:          64,
-	MD5:          16,
-}
-
 func uvarint(buf []byte) (uint64, []byte, error) {
 	n, c, err := varint.FromUvarint(buf)
 	if err != nil {
@@ -231,15 +207,11 @@ func FromB58String(s string) (m Multihash, err error) {
 // Cast casts a buffer onto a multihash, and returns an error
 // if it does not work.
 func Cast(buf []byte) (Multihash, error) {
-	dm, err := Decode(buf)
+	_, err := Decode(buf)
 	if err != nil {
 		return Multihash{}, err
 	}
 
-	if !ValidCode(dm.Code) {
-		return Multihash{}, ErrUnknownCode
-	}
-
 	return Multihash(buf), nil
 }
 
@@ -266,11 +238,10 @@ func Decode(buf []byte) (*DecodedMultihash, error) {
 
 // Encode a hash digest along with the specified function code.
 // Note: the length is derived from the length of the digest itself.
+//
+// The error return is legacy; it is always nil.
 func Encode(buf []byte, code uint64) ([]byte, error) {
-	if !ValidCode(code) {
-		return nil, ErrUnknownCode
-	}
-
+	// FUTURE: this function always causes heap allocs... but when used, this value is almost always going to be appended to another buffer (either as part of CID creation, or etc) -- should this whole function be rethought and alternatives offered?
 	newBuf := make([]byte, varint.UvarintSize(code)+varint.UvarintSize(uint64(len(buf)))+len(buf))
 	n := varint.PutUvarint(newBuf, code)
 	n += varint.PutUvarint(newBuf[n:], uint64(len(buf)))
@@ -285,12 +256,6 @@ func EncodeName(buf []byte, name string) ([]byte, error) {
 	return Encode(buf, Names[name])
 }
 
-// ValidCode checks whether a multihash code is valid.
-func ValidCode(code uint64) bool {
-	_, ok := Codes[code]
-	return ok
-}
-
 // readMultihashFromBuf reads a multihash from the given buffer, returning the
 // individual pieces of the multihash.
 // Note: the returned digest is a slice over the passed in data and should be

multihash/main.go

@@ -8,6 +8,8 @@ import (
 
 	mh "github.com/multiformats/go-multihash"
 	mhopts "github.com/multiformats/go-multihash/opts"
+	_ "github.com/multiformats/go-multihash/register/all"
+	_ "github.com/multiformats/go-multihash/register/miniosha256"
 )
 
 var usage = `usage: %s [options] [FILE]

multihash_test.go

@@ -224,16 +224,6 @@ func ExampleDecode() {
 	// obj: sha1 0x11 20 0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33
 }
 
-func TestValidCode(t *testing.T) {
-	for i := uint64(0); i < 0xff; i++ {
-		_, ok := tCodes[i]
-
-		if ValidCode(i) != ok {
-			t.Error("ValidCode incorrect for: ", i)
-		}
-	}
-}
-
 func TestCast(t *testing.T) {
 	for _, tc := range testCases {
 		ob, err := hex.DecodeString(tc.hex)

opts/opts.go

@@ -110,8 +110,13 @@ func (o *Options) ParseError() error {
 		}
 		o.Length = o.Length / 8
 
-		if o.Length > mh.DefaultLengths[o.AlgorithmCode] {
-			o.Length = mh.DefaultLengths[o.AlgorithmCode]
+		h, _ := mh.GetHasher(o.AlgorithmCode)
+		hsize := 0
+		if h != nil {
+			hsize = h.Size()
+		}
+		if o.Length > hsize {
+			o.Length = hsize
 		}
 	}
 	return nil

register/all/multihash_all.go

@@ -0,0 +1,22 @@
+/*
+	This package has no purpose except to perform registration of mulithashes.
+
+	It is meant to be used as a side-effecting import, e.g.
+
+		import (
+			_ "github.com/multiformats/go-multihash/register/all"
+		)
+
+	This package registers many multihashes at once.
+	Importing it will increase the size of your dependency tree significantly.
+	It's recommended that you import this package if you're building some
+	kind of data broker application, which may need to handle many different kinds of hashes;
+	if you're building an application which you know only handles a specific hash,
+	importing this package may bloat your builds unnecessarily.
+*/
+package all
+
+import (
+	_ "github.com/multiformats/go-multihash/register/blake2"
+	_ "github.com/multiformats/go-multihash/register/sha3"
+)