Merge pull request #141 from ShyftNetwork/db-upstream

Will merge this and then move to staging where @GregTheGreek can approve and create the next release.

Author: dbkbali

.gitignore

@@ -27,7 +27,6 @@ ethash/
 /build/bin/
 /geth*.zip
 
-
 # travis
 profile.tmp
 profile.cov
@@ -49,8 +48,6 @@ profile.cov
 /dashboard/assets/package-lock.json
 
 **/yarn-error.log
-$HOME
-
 # Remove Shyft Data
 shyftData/*
 shyft-cli/web3/token_test/node_modules

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "tests"]
-	path = tests/testdata
-	url = https://github.com/ShyftNetwork/shyft_tests

.travis.yml

@@ -66,7 +66,4 @@ before_script:
 
 script:
 - go run build/ci.go install
-- go run build/ci.go test -coverage $TEST_PACKAGES
-
-
-
+- go run build/ci.go test -coverage $TEST_PACKAGES

README.md

@@ -71,6 +71,7 @@ To stop Geth, **`crtl+C`** in the terminal window, if you proceed with the start
 ``./shyft-geth.sh --start`` Starts GETH
 
 To see transactions being submitted on the network see the sendTransactions command in the CLI section of this readme.
+
 #### Docker Images
 
 Two sets of Docker Images are available for ShyftGeth, the Postgresql Database, and the Shyft Blockchain Explorer, which can be used for local development and testnet connection. The development settings are included in docker-compose.yml, the testnet settings are included in docker-compose.production.yml. To launch these containers you will need to have docker-compose installed on your computer. Installation instructions for docker-compose are available [here](https://docs.docker.com/install/).
@@ -94,6 +95,7 @@ To stop/pause mining - enter:
 **`docker-compose stop`**
 
 And then just issue `docker-compose up` to continue mining.
+
 #### Docker Postgresql - DB Connection
 From your local machine you can view the database by connecting to the database in the container at 
 **``127.0.0.1:8001``**
@@ -225,7 +227,7 @@ For prerequisites and detailed build instructions please read the
 [Installation Instructions](https://github.com/empyrean/go-ethereum/wiki/Building-Ethereum)
 on the wiki.
 
-Building geth requires both a Go (version 1.7 or later) and a C compiler.
+Building geth requires both a Go (version 1.9 or later) and a C compiler.
 You can install them using your favourite package manager.
 Once the dependencies are installed, run
 
@@ -375,7 +377,7 @@ HTTP based JSON-RPC API options:
 - `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it)
 
 You'll need to use your own programming environments' capabilities (libraries, tools, etc) to connect
-via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](http://www.jsonrpc.org/specification)
+via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](https://www.jsonrpc.org/specification)
 on all transports. You can reuse the same connection for multiple requests!
 
 **Note: Please understand the security implications of opening up an HTTP/WS based transport before
@@ -487,6 +489,77 @@ Which will start mining blocks and transactions on a single CPU thread, creditin
 the account specified by `--etherbase`. You can further tune the mining by changing the default gas
 limit blocks converge to (`--targetgaslimit`) and the price transactions are accepted at (`--gasprice`).
 
+## SHYFT NOTES
+
+#### CLI
+
+Run `./shyft-geth.sh` with one of the following flags:
+
+- `--setup` - Setups postgres and the shyft chain db.
+- `--start` - Starts geth.
+- `--reset` - Drops postgress and chain db, and reinstantiates both.
+- `--js [web3 filename]` - Executes web3 calls with a passed file name. If the file name is `sendTransactions.js`, `./shyft-geth.sh --js sendTransactions`.
+
+#### Docker Images
+
+Docker Images are available for ShyftGeth and the Postgresql Database which can be used for development and testing. To launch these containers you will need to have docker-compose installed on your computer. Installation instructions for docker-compose are available [here](https://docs.docker.com/install/).
+
+To launch ShyftGeth, PG, the ShyftBlock Explorer Api and UI - issue the following command from the root of the project directory:
+
+`docker-compose up`
+
+If you would like to reinitialize/rebuild the docker images you can issue the following command:
+
+`docker-compose up --build`
+
+To rebuild any one of the services - issue the following commands:
+
+```
+docker-compose up -d --no-deps --build <docker compose file service name> 
+
+# ie. for shyftBlockExplorerApi:
+# docker-compose up -d --no-deps --build shyft_block_api
+```
+__The Postgresql Database Container will persist the database data to a folder in the root of the project directory - pg-data" __. So if you do want to reinitialize the database you should delete this docker container prior to launching the docker containers. To delete this docker volume and have it recreated you should input the following command:
+
+```docker volume rm go-empyrean_pg-data```
+
+From your local machine you can view the database by connecting to the database in the container at 127.0.0.1:8001. To access the shyftBlockExplorer open a browser and visit http://localhost:3000
+
+__Blockchain data is persisted to ./ethash/.ethash__ and ./shyftData. If you would like to reset the test blockchain you will need to delete the ./ethash and ./shyftData directories.
+
+The docker container for the ShyftBlockExplorerApi utilizes govendor to minimize its image size. __If you would like the docker image for this container to reflect any uncommitted changes which may have occurred in the go-empyrean repository, ie. changes with respect to go-empyrean core (ie. cryptographic functions and database). Prior to launching the docker containers you should rebuild the vendor directory for the shyftBlockExplorerApi - by executing the following steps:__
+
+```
+# remove existing shyftBlockExplorerApi vendor.json and vendored components:
+
+rm -rf shyftBlockExplorerApi/vendor
+
+# reinitialize vendor.json
+
+cd shyftBlockExplorerApi && govendor init
+
+# rebuild vendor.json using latest uncommitted changes
+
+govendor add -tree -uncommitted +external
+
+# due to a bug in govendor and it not being able to pull in some dependencies that are c-header files 
+# you should execute the following commands - see these issues - which whilst closed
+# appears to have not been fixed: https://github.com/kardianos/govendor/issues/124 && https://github.com/kardianos/govendor/issues/61
+
+govendor remove github.com/ShyftNetwork/go-empyrean/crypto/secp256k1/^
+govendor fetch github.com/ShyftNetwork/go-empyrean/crypto/secp256k1/^
+
+```
+
+NB: The Shyft Geth docker image size is 1+ GB so make sure you have adequate space on your disk drive/
+
+_TODO_
+
+- Find better dependency management solution that pulls in c header files without manual intervention
+- Reduce size of the ShytfGeth docker container which is responsible for mining and running the blockchain
+- Adjust docker scripts and ports to facilitate sending of test transactions
+- Modify Docker scripts to facilitate hot reloading during development
 
 ## Contribution
 

accounts/abi/argument.go

@@ -243,11 +243,7 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
 	// input offset is the bytes offset for packed output
 	inputOffset := 0
 	for _, abiArg := range abiArgs {
-		if abiArg.Type.T == ArrayTy {
-			inputOffset += 32 * abiArg.Type.Size
-		} else {
-			inputOffset += 32
-		}
+		inputOffset += getDynamicTypeOffset(abiArg.Type)
 	}
 	var ret []byte
 	for i, a := range args {
@@ -257,14 +253,13 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
 		if err != nil {
 			return nil, err
 		}
-		// check for a slice type (string, bytes, slice)
-		if input.Type.requiresLengthPrefix() {
-			// calculate the offset
-			offset := inputOffset + len(variableInput)
+		// check for dynamic types
+		if isDynamicType(input.Type) {
 			// set the offset
-			ret = append(ret, packNum(reflect.ValueOf(offset))...)
-			// Append the packed output to the variable input. The variable input
-			// will be appended at the end of the input.
+			ret = append(ret, packNum(reflect.ValueOf(inputOffset))...)
+			// calculate next offset
+			inputOffset += len(packed)
+			// append to variable input
 			variableInput = append(variableInput, packed...)
 		} else {
 			// append the packed value to the input

accounts/abi/pack_test.go

@@ -324,6 +324,66 @@ func TestPack(t *testing.T) {
 			"foobar",
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000006666f6f6261720000000000000000000000000000000000000000000000000000"),
 		},
+		{
+			"string[]",
+			[]string{"hello", "foobar"},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2
+				"0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"0000000000000000000000000000000000000000000000000000000000000080" + // offset 128 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5
+				"68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0]
+				"0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6
+				"666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1]
+		},
+		{
+			"string[2]",
+			[]string{"hello", "foobar"},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset to i = 0
+				"0000000000000000000000000000000000000000000000000000000000000080" + // offset to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5
+				"68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0]
+				"0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6
+				"666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1]
+		},
+		{
+			"bytes32[][]",
+			[][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2
+				"0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2
+				"0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
+
+		{
+			"bytes32[][2]",
+			[][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2
+				"0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
+
+		{
+			"bytes32[3][2]",
+			[][]common.Hash{{{1}, {2}, {3}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[0][2]
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil {
@@ -336,7 +396,7 @@ func TestPack(t *testing.T) {
 		}
 
 		if !bytes.Equal(output, test.output) {
-			t.Errorf("%d failed. Expected bytes: '%x' Got: '%x'", i, test.output, output)
+			t.Errorf("input %d for typ: %v failed. Expected bytes: '%x' Got: '%x'", i, typ.String(), test.output, output)
 		}
 	}
 }

accounts/abi/type.go

@@ -183,27 +183,67 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 		return nil, err
 	}
 
-	if t.T == SliceTy || t.T == ArrayTy {
-		var packed []byte
+	switch t.T {
+	case SliceTy, ArrayTy:
+		var ret []byte
 
+		if t.requiresLengthPrefix() {
+			// append length
+			ret = append(ret, packNum(reflect.ValueOf(v.Len()))...)
+		}
+
+		// calculate offset if any
+		offset := 0
+		offsetReq := isDynamicType(*t.Elem)
+		if offsetReq {
+			offset = getDynamicTypeOffset(*t.Elem) * v.Len()
+		}
+		var tail []byte
 		for i := 0; i < v.Len(); i++ {
 			val, err := t.Elem.pack(v.Index(i))
 			if err != nil {
 				return nil, err
 			}
-			packed = append(packed, val...)
-		}
-		if t.T == SliceTy {
-			return packBytesSlice(packed, v.Len()), nil
-		} else if t.T == ArrayTy {
-			return packed, nil
+			if !offsetReq {
+				ret = append(ret, val...)
+				continue
+			}
+			ret = append(ret, packNum(reflect.ValueOf(offset))...)
+			offset += len(val)
+			tail = append(tail, val...)
 		}
+		return append(ret, tail...), nil
+	default:
+		return packElement(t, v), nil
 	}
-	return packElement(t, v), nil
 }
 
 // requireLengthPrefix returns whether the type requires any sort of length
 // prefixing.
 func (t Type) requiresLengthPrefix() bool {
 	return t.T == StringTy || t.T == BytesTy || t.T == SliceTy
 }
+
+// isDynamicType returns true if the type is dynamic.
+// StringTy, BytesTy, and SliceTy(irrespective of slice element type) are dynamic types
+// ArrayTy is considered dynamic if and only if the Array element is a dynamic type.
+// This function recursively checks the type for slice and array elements.
+func isDynamicType(t Type) bool {
+	// dynamic types
+	// array is also a dynamic type if the array type is dynamic
+	return t.T == StringTy || t.T == BytesTy || t.T == SliceTy || (t.T == ArrayTy && isDynamicType(*t.Elem))
+}
+
+// getDynamicTypeOffset returns the offset for the type.
+// See `isDynamicType` to know which types are considered dynamic.
+// If the type t is an array and element type is not a dynamic type, then we consider it a static type and
+// return 32 * size of array since length prefix is not required.
+// If t is a dynamic type or element type(for slices and arrays) is dynamic, then we simply return 32 as offset.
+func getDynamicTypeOffset(t Type) int {
+	// if it is an array and there are no dynamic types
+	// then the array is static type
+	if t.T == ArrayTy && !isDynamicType(*t.Elem) {
+		return 32 * t.Size
+	}
+	return 32
+}

accounts/keystore/account_cache.go

@@ -266,7 +266,10 @@ func (ac *accountCache) scanAccounts() error {
 		case (addr == common.Address{}):
 			log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address")
 		default:
-			return &accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}}
+			return &accounts.Account{
+				Address: addr,
+				URL:     accounts.URL{Scheme: KeyStoreScheme, Path: path},
+			}
 		}
 		return nil
 	}

accounts/keystore/key.go

@@ -171,7 +171,10 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
 	if err != nil {
 		return nil, accounts.Account{}, err
 	}
-	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}}
+	a := accounts.Account{
+		Address: key.Address,
+		URL:     accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))},
+	}
 	if err := ks.StoreKey(a.URL.Path, key, auth); err != nil {
 		zeroKey(key.PrivateKey)
 		return nil, a, err
@@ -224,5 +227,6 @@ func toISO8601(t time.Time) string {
 	} else {
 		tz = fmt.Sprintf("%03d00", offset/3600)
 	}
-	return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
+	return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s",
+		t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
 }

accounts/keystore/keystore_passphrase.go renamed to accounts/keystore/passphrase.go

@@ -233,6 +233,7 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
 		PrivateKey: key,
 	}, nil
 }
+
 func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) {
 	if cryptoJson.Cipher != "aes-128-ctr" {
 		return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher)