Improve composite key Bytes/FromBytes and make tools (#335)

* bls

* composite

* abci

* fix TestBroadcastEvidence_DuplicateVoteEvidence, see tendermint/tendermint#6678

* (fix) Add comments

* (fix) Use `os.UserHomeDir` and `filepath.Join`

abci/cmd/abci-cli/abci-cli.go

@@ -2,13 +2,17 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 
+	"github.com/line/ostracon/config"
+
 	"github.com/spf13/cobra"
 
 	"github.com/line/ostracon/libs/log"
@@ -22,6 +26,7 @@ import (
 	servertest "github.com/line/ostracon/abci/tests/server"
 	"github.com/line/ostracon/abci/types"
 	"github.com/line/ostracon/abci/version"
+	"github.com/line/ostracon/crypto/encoding"
 	"github.com/line/ostracon/proto/ostracon/crypto"
 )
 
@@ -49,6 +54,9 @@ var (
 
 	// kvstore
 	flagPersist string
+
+	// staking power for make validator_tx
+	flagStakingPower int64
 )
 
 var RootCmd = &cobra.Command{
@@ -136,11 +144,18 @@ func addQueryFlags() {
 }
 
 func addCounterFlags() {
-	counterCmd.PersistentFlags().BoolVarP(&flagSerial, "serial", "", false, "enforce incrementing (serial) transactions")
+	counterCmd.PersistentFlags().BoolVarP(&flagSerial, "serial", "", false,
+		"enforce incrementing (serial) transactions")
 }
 
 func addKVStoreFlags() {
-	kvstoreCmd.PersistentFlags().StringVarP(&flagPersist, "persist", "", "", "directory to use for a database")
+	kvstoreCmd.PersistentFlags().StringVarP(&flagPersist, "persist", "", "",
+		"directory to use for a database")
+}
+
+func addPersistKVStoreMakeValSetChangeTxFlags() {
+	kvstoreCmd.PersistentFlags().Int64VarP(&flagStakingPower, "staking_power", "p", int64(10),
+		"stakin power for ValSetChangeTx")
 }
 
 func addCommands() {
@@ -162,6 +177,10 @@ func addCommands() {
 	RootCmd.AddCommand(counterCmd)
 	addKVStoreFlags()
 	RootCmd.AddCommand(kvstoreCmd)
+
+	// for examples of persist_kvstore
+	addPersistKVStoreMakeValSetChangeTxFlags()
+	RootCmd.AddCommand(persistKvstoreMakeValSetChangeTxCmd)
 }
 
 var batchCmd = &cobra.Command{
@@ -269,20 +288,28 @@ var queryCmd = &cobra.Command{
 
 var counterCmd = &cobra.Command{
 	Use:   "counter",
-	Short: "ABCI demo example",
-	Long:  "ABCI demo example",
+	Short: "ABCI demo example - counter",
+	Long:  "ABCI demo example - counter",
 	Args:  cobra.ExactArgs(0),
 	RunE:  cmdCounter,
 }
 
 var kvstoreCmd = &cobra.Command{
 	Use:   "kvstore",
-	Short: "ABCI demo example",
-	Long:  "ABCI demo example",
+	Short: "ABCI demo example - kvstore",
+	Long:  "ABCI demo example - kvstore",
 	Args:  cobra.ExactArgs(0),
 	RunE:  cmdKVStore,
 }
 
+var persistKvstoreMakeValSetChangeTxCmd = &cobra.Command{
+	Use:   "valset_change_tx",
+	Short: "persist_kvstore - make ValSetChangeTx",
+	Long:  "persist_kvstore - make ValSetChangeTx",
+	Args:  cobra.ExactArgs(0),
+	RunE:  cmdPersistKVStoreMakeValSetChangeTx,
+}
+
 var testCmd = &cobra.Command{
 	Use:   "test",
 	Short: "run integration tests",
@@ -685,6 +712,61 @@ func cmdKVStore(cmd *cobra.Command, args []string) error {
 	select {}
 }
 
+func cmdPersistKVStoreMakeValSetChangeTx(cmd *cobra.Command, args []string) error {
+	c := config.DefaultConfig()
+	userHome, err := os.UserHomeDir()
+	if err != nil {
+		panic(err)
+	}
+	c.SetRoot(filepath.Join(userHome, config.DefaultOstraconDir))
+	keyFilePath := c.PrivValidatorKeyFile()
+	if !tmos.FileExists(keyFilePath) {
+		return fmt.Errorf("private validator file %s does not exist", keyFilePath)
+	}
+	keyFile, err := kvstore.LoadPrivValidatorKeyFile(keyFilePath)
+	if err != nil {
+		panic(err)
+	}
+	publicKey, err := encoding.PubKeyToProto(keyFile.PubKey)
+	if err != nil {
+		panic(err)
+	}
+	pubStr, tx := kvstore.MakeValSetChangeTxAndMore(publicKey, flagStakingPower)
+	{
+		fmt.Printf("DeliverTxSync: data=%s, tx=%s\n", pubStr, tx)
+		res, err := client.DeliverTxSync(types.RequestDeliverTx{Tx: []byte(tx)})
+		if err != nil {
+			return err
+		}
+		printResponse(cmd, args, response{
+			Code: res.Code,
+			Data: res.Data,
+			Info: res.Info,
+			Log:  res.Log,
+		})
+	}
+	{
+		fmt.Printf("QuerySync: data=%s\n", pubStr)
+		res, err := client.QuerySync(types.RequestQuery{Path: "/val", Data: []byte(pubStr)})
+		if err != nil {
+			return err
+		}
+		printResponse(cmd, args, response{
+			Code: res.Code,
+			Info: res.Info,
+			Log:  res.Log,
+		})
+		fmt.Printf("original:publicKey:%s\n", publicKey)
+		validatorUpdate := types.ValidatorUpdate{}
+		err = types.ReadMessage(bytes.NewReader(res.Value), &validatorUpdate)
+		if err != nil {
+			panic(err)
+		}
+		fmt.Printf("saved   :publicKey:%s\n", validatorUpdate.PubKey)
+	}
+	return nil
+}
+
 //--------------------------------------------------------------------------------
 
 func printResponse(cmd *cobra.Command, args []string, rsp response) {
@@ -702,7 +784,7 @@ func printResponse(cmd *cobra.Command, args []string, rsp response) {
 	}
 
 	if len(rsp.Data) != 0 {
-		// Do no print this line when using the commit command
+		// Do not print this line when using the commit command
 		// because the string comes out as gibberish
 		if cmd.Use != "commit" {
 			fmt.Printf("-> data: %s\n", rsp.Data)

abci/example/kvstore/helpers.go

@@ -1,13 +1,33 @@
 package kvstore
 
 import (
+	"fmt"
+	"io/ioutil"
+
 	"github.com/line/ostracon/abci/types"
 	"github.com/line/ostracon/crypto"
 	"github.com/line/ostracon/crypto/composite"
+	tmjson "github.com/line/ostracon/libs/json"
+	tmos "github.com/line/ostracon/libs/os"
 	tmrand "github.com/line/ostracon/libs/rand"
+	"github.com/line/ostracon/privval"
 )
 
-// Generates a default private key for use in an example or test.
+// LoadPrivValidatorKeyFile Load private key for use in an example or test.
+func LoadPrivValidatorKeyFile(keyFilePath string) (*privval.FilePVKey, error) {
+	if !tmos.FileExists(keyFilePath) {
+		return nil, fmt.Errorf("private validator file %s does not exist", keyFilePath)
+	}
+	keyJSONBytes, _ := ioutil.ReadFile(keyFilePath)
+	pvKey := privval.FilePVKey{}
+	err := tmjson.Unmarshal(keyJSONBytes, &pvKey)
+	if err != nil {
+		return nil, fmt.Errorf("error reading PrivValidator key from %v: %v", keyFilePath, err)
+	}
+	return &pvKey, nil
+}
+
+// GenDefaultPrivKey Generates a default private key for use in an example or test.
 func GenDefaultPrivKey() crypto.PrivKey {
 	return composite.GenPrivKey()
 }

abci/example/kvstore/helpers_test.go

@@ -0,0 +1,43 @@
+package kvstore
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/line/ostracon/privval"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadPrivValidatorKeyFile(t *testing.T) {
+	tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_")
+	require.Nil(t, err)
+	tempStateFile, err := ioutil.TempFile("", "priv_validator_state_")
+	require.Nil(t, err)
+
+	{
+		// does not exist
+		_, err := LoadPrivValidatorKeyFile("DOES_NOT_EXIST")
+		require.NotNil(t, err)
+		require.Contains(t, err.Error(), "does not exist")
+	}
+
+	{
+		// error reading since empty
+		_, err := LoadPrivValidatorKeyFile(tempKeyFile.Name())
+		require.NotNil(t, err)
+		require.Contains(t, err.Error(), "error reading")
+	}
+
+	expected, err := privval.GenFilePV(tempKeyFile.Name(), tempStateFile.Name(), privval.PrivKeyTypeEd25519)
+	require.Nil(t, err)
+
+	expected.Save()
+
+	// success
+	actual, err := LoadPrivValidatorKeyFile(tempKeyFile.Name())
+	require.Nil(t, err)
+	assert.Equal(t, expected.Key.Address, actual.Address)
+	assert.Equal(t, expected.Key.PrivKey, actual.PrivKey)
+	assert.Equal(t, expected.Key.PubKey, actual.PubKey)
+}

abci/example/kvstore/kvstore_test.go

@@ -6,6 +6,8 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/line/ostracon/libs/log"
@@ -182,6 +184,9 @@ func TestValUpdates(t *testing.T) {
 	vals2 = kvstore.Validators()
 	valsEqual(t, vals1, vals2)
 
+	for _, v := range vals2 {
+		existInPersistStore(t, kvstore, v)
+	}
 }
 
 func makeApplyBlock(
@@ -210,6 +215,27 @@ func makeApplyBlock(
 
 }
 
+func existInPersistStore(t *testing.T, kvstore types.Application, v types.ValidatorUpdate) {
+	// success
+	pubkeyStr, _ := MakeValSetChangeTxAndMore(v.PubKey, v.Power)
+	resQuery := kvstore.Query(types.RequestQuery{Path: "/val", Data: []byte(pubkeyStr)})
+	assert.False(t, resQuery.IsErr(), resQuery)
+	assert.Regexp(t, "^key=.+, validatorUpdate.PubKey=.+, validatorUpdate.Power=.+$", resQuery.Log)
+	// failures
+	{
+		// default Query: does not exist
+		r := kvstore.Query(types.RequestQuery{Path: "/val_", Data: []byte(pubkeyStr)})
+		assert.False(t, r.IsErr(), r)
+		assert.Contains(t, r.Log, "does not exist")
+	}
+	{
+		// Query: does not exist
+		r := kvstore.Query(types.RequestQuery{Path: "/val", Data: []byte{}})
+		assert.False(t, r.IsErr(), r)
+		assert.Contains(t, r.Log, "cannot get")
+	}
+}
+
 // order doesn't matter
 func valsEqual(t *testing.T, vals1, vals2 []types.ValidatorUpdate) {
 	if len(vals1) != len(vals2) {

abci/example/kvstore/persistent_kvstore.go

@@ -113,6 +113,19 @@ func (app *PersistentKVStoreApplication) Query(reqQuery types.RequestQuery) (res
 
 		resQuery.Key = reqQuery.Data
 		resQuery.Value = value
+
+		if value == nil {
+			resQuery.Log = "cannot get"
+			return
+		}
+		validatorUpdate := types.ValidatorUpdate{}
+		err = types.ReadMessage(bytes.NewReader(resQuery.Value), &validatorUpdate)
+		if err != nil {
+			panic(err)
+		}
+		pubKey, _ := cryptoenc.PubKeyFromProto(&validatorUpdate.PubKey)
+		resQuery.Log = fmt.Sprintf("key=%s, validatorUpdate.PubKey=%v, validatorUpdate.Power=%d",
+			resQuery.Key, pubKey, validatorUpdate.Power)
 		return
 	default:
 		return app.app.Query(reqQuery)
@@ -206,20 +219,26 @@ func (app *PersistentKVStoreApplication) Validators() (validators []types.Valida
 }
 
 func MakeValSetChangeTx(pubkey pc.PublicKey, power int64) []byte {
+	_, tx := MakeValSetChangeTxAndMore(pubkey, power)
+	return []byte(tx)
+}
+
+func MakeValSetChangeTxAndMore(pubkey pc.PublicKey, power int64) (string, string) {
 	pkBytes, err := pubkey.Marshal()
 	if err != nil {
 		panic(err)
 	}
 	pubStr := base64.StdEncoding.EncodeToString(pkBytes)
-	return []byte(fmt.Sprintf("val:%s!%d", pubStr, power))
+	return pubStr, fmt.Sprintf("val:%s!%d", pubStr, power)
 }
 
 func isValidatorTx(tx []byte) bool {
 	return strings.HasPrefix(string(tx), ValidatorSetChangePrefix)
 }
 
 // format is "val:pubkey!power"
-// pubkey is a base64-encoded 32-byte ed25519 key
+// pubkey is a base64-encoded proto.ostracon.crypto.PublicKey bytes
+// See MakeValSetChangeTx
 func (app *PersistentKVStoreApplication) execValidatorTx(tx []byte) types.ResponseDeliverTx {
 	tx = tx[len(ValidatorSetChangePrefix):]
 
@@ -237,20 +256,20 @@ func (app *PersistentKVStoreApplication) execValidatorTx(tx []byte) types.Respon
 	if err != nil {
 		return types.ResponseDeliverTx{
 			Code: code.CodeTypeEncodingError,
-			Log:  fmt.Sprintf("Pubkey (%s) is invalid base64", pubkeyS)}
+			Log:  fmt.Sprintf("pubkeyS (%s) is invalid base64", pubkeyS)}
 	}
 	var pkProto pc.PublicKey
 	err = pkProto.Unmarshal(pkBytes)
 	if err != nil {
 		return types.ResponseDeliverTx{
 			Code: code.CodeTypeEncodingError,
-			Log:  fmt.Sprintf("Pubkey (%s) is invalid binary", pubkeyS)}
+			Log:  fmt.Sprintf("pkBytes (%x) is invalid binary", pkBytes)}
 	}
 	pubkey, err := cryptoenc.PubKeyFromProto(&pkProto)
 	if err != nil {
 		return types.ResponseDeliverTx{
 			Code: code.CodeTypeEncodingError,
-			Log:  fmt.Sprintf("Pubkey (%s) is invalid binary", pubkeyS)}
+			Log:  fmt.Sprintf("pkProto (%s) is invalid binary", pkProto)}
 	}
 
 	// decode the power
@@ -266,14 +285,16 @@ func (app *PersistentKVStoreApplication) execValidatorTx(tx []byte) types.Respon
 }
 
 // add, update, or remove a validator
+// See MakeValSetChangeTx
 func (app *PersistentKVStoreApplication) updateValidator(v types.ValidatorUpdate) types.ResponseDeliverTx {
 	pubkey, err := cryptoenc.PubKeyFromProto(&v.PubKey)
 	if err != nil {
 		return types.ResponseDeliverTx{
 			Code: code.CodeTypeEncodingError,
 			Log:  fmt.Sprintf("Error encoding Public Key: %s", err)}
 	}
-	key := []byte("val:" + string(pubkey.Bytes()))
+	pubStr, _ := MakeValSetChangeTxAndMore(v.PubKey, v.Power)
+	key := []byte("val:" + pubStr)
 
 	if v.Power == 0 {
 		// remove validator
@@ -282,7 +303,6 @@ func (app *PersistentKVStoreApplication) updateValidator(v types.ValidatorUpdate
 			panic(err)
 		}
 		if !hasKey {
-			pubStr := base64.StdEncoding.EncodeToString(pubkey.Bytes())
 			return types.ResponseDeliverTx{
 				Code: code.CodeTypeUnauthorized,
 				Log:  fmt.Sprintf("Cannot remove non-existent validator %s", pubStr)}