fix decodeBytes() integer overflow on 32-bit systems

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+### Bug Fixes
+
+- Fix integer overflow in `decodeBytes()` that can cause out-of-memory errors on 32-bit machines for certain inputs.
+
 ## 0.15.0 (November 23, 2020)
 
 The IAVL project has moved from https://github.com/tendermint/iavl to

encoding.go

@@ -16,9 +16,14 @@ func decodeBytes(bz []byte) ([]byte, int, error) {
 	if err != nil {
 		return nil, n, err
 	}
+	// check for overflows
 	if int(size) < 0 {
 		return nil, n, fmt.Errorf("invalid negative length %v decoding []byte", size)
 	}
+	// ^uint(0) >> 1 will help determine the max int value variably on 32-bit and 64-bit machines.
+	if uint64(n)+size >= uint64(^uint(0)>>1) {
+		return nil, n, fmt.Errorf("invalid out of range length %v decoding []byte", uint64(n)+size)
+	}
 	if len(bz) < n+int(size) {
 		return nil, n, fmt.Errorf("insufficient bytes decoding []byte of length %v", size)
 	}