Use the Trillian testing environment (#727)

* Use Trillian MapEnv for integration tests

* Use Trillian Verifier

* Use maphasher in verifier

* Compute leafHash before passing to inclusion proof verification

* use real public key from tree

* Adjust tests for off-by-one TreeRevision starts at 1 now

* Fixups

Author: gdbelvin

cmd/keytransparency-client/grpcc/grpc_client.go

@@ -31,11 +31,10 @@ import (
 	"github.com/google/keytransparency/core/crypto/vrf"
 	"github.com/google/keytransparency/core/mutator"
 	"github.com/google/keytransparency/core/mutator/entry"
-	"github.com/google/keytransparency/core/tree/sparse"
-	tv "github.com/google/keytransparency/core/tree/sparse/verifier"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/google/trillian/client"
+	"github.com/google/trillian/merkle/maphasher"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 
@@ -99,7 +98,7 @@ func New(
 	return &Client{
 		cli:        client,
 		vrf:        vrf,
-		kt:         kt.New(vrf, tv.New(sparse.CONIKSHasher), verifier, log),
+		kt:         kt.New(vrf, maphasher.Default, verifier, log),
 		log:        log,
 		mutator:    entry.New(),
 		RetryCount: 1,

cmd/keytransparency-server/main.go

@@ -17,7 +17,6 @@ package main
 import (
 	"database/sql"
 	"flag"
-	"fmt"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -27,11 +26,9 @@ import (
 	"github.com/google/keytransparency/core/crypto/vrf"
 	"github.com/google/keytransparency/core/crypto/vrf/p256"
 	"github.com/google/keytransparency/core/keyserver"
-	"github.com/google/keytransparency/core/mapserver"
 	"github.com/google/keytransparency/core/mutator/entry"
 
 	cmutation "github.com/google/keytransparency/core/mutation"
-	ctxn "github.com/google/keytransparency/core/transaction"
 	"github.com/google/keytransparency/impl/authorization"
 	gauth "github.com/google/keytransparency/impl/google/authentication"
 	"github.com/google/keytransparency/impl/mutation"
@@ -40,8 +37,6 @@ import (
 	"github.com/google/keytransparency/impl/sql/commitments"
 	"github.com/google/keytransparency/impl/sql/engine"
 	"github.com/google/keytransparency/impl/sql/mutations"
-	"github.com/google/keytransparency/impl/sql/sequenced"
-	"github.com/google/keytransparency/impl/sql/sqlhist"
 	"github.com/google/keytransparency/impl/transaction"
 	"github.com/google/trillian"
 
@@ -131,18 +126,6 @@ func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Ha
 	})
 }
 
-func newReadonlyMapServer(ctx context.Context, mapID int64, sqldb *sql.DB, factory ctxn.Factory) (trillian.TrillianMapClient, error) {
-	tree, err := sqlhist.New(ctx, mapID, factory)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to create SQL history: %v", err)
-	}
-	sths, err := sequenced.New(sqldb, mapID)
-	if err != nil {
-		return nil, fmt.Errorf("sequenced.New(%v): %v", mapID, err)
-	}
-	return mapserver.NewReadonly(mapID, tree, factory, sths), nil
-}
-
 func main() {
 	flag.Parse()
 
@@ -192,22 +175,12 @@ func main() {
 	tlog := trillian.NewTrillianLogClient(tconn)
 
 	// Connect to map server.
-	var tmap trillian.TrillianMapClient
-	var tadmin trillian.TrillianAdminClient
-	if *mapURL != "" {
-		mconn, err := grpc.Dial(*mapURL, grpc.WithInsecure())
-		if err != nil {
-			glog.Exitf("grpc.Dial(%v): %v", *mapURL, err)
-		}
-		tmap = trillian.NewTrillianMapClient(mconn)
-		tadmin = trillian.NewTrillianAdminClient(mconn)
-	} else {
-		// Create an in-process readonly mapserver.
-		tmap, err = newReadonlyMapServer(context.Background(), *mapID, sqldb, factory)
-		if err != nil {
-			glog.Exitf("newReadonlyMapServer(): %v", err)
-		}
+	mconn, err := grpc.Dial(*mapURL, grpc.WithInsecure())
+	if err != nil {
+		glog.Exitf("grpc.Dial(%v): %v", *mapURL, err)
 	}
+	tmap := trillian.NewTrillianMapClient(mconn)
+	tadmin := trillian.NewTrillianAdminClient(mconn)
 
 	// Create gRPC server.
 	svr := keyserver.New(*logID, tlog, *mapID, tmap, tadmin, commitments,

cmd/keytransparency-signer/main.go

@@ -17,28 +17,21 @@ package main
 import (
 	"database/sql"
 	"flag"
-	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/google/keytransparency/core/admin"
 	"github.com/google/keytransparency/core/appender"
-	"github.com/google/keytransparency/core/mapserver"
 	"github.com/google/keytransparency/core/mutator/entry"
 	"github.com/google/keytransparency/core/signer"
-	ctxn "github.com/google/keytransparency/core/transaction"
 	"github.com/google/keytransparency/impl/config"
 	"github.com/google/keytransparency/impl/sql/engine"
 	"github.com/google/keytransparency/impl/sql/mutations"
-	"github.com/google/keytransparency/impl/sql/sequenced"
-	"github.com/google/keytransparency/impl/sql/sqlhist"
 	"github.com/google/keytransparency/impl/transaction"
 
 	"github.com/golang/glog"
 	"github.com/google/trillian"
-	"github.com/google/trillian/crypto/keys"
 	_ "github.com/google/trillian/merkle/objhasher" // Register objhasher
-	"github.com/google/trillian/util"
 	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
@@ -76,25 +69,6 @@ func openDB() *sql.DB {
 	return db
 }
 
-func newMapServer(ctx context.Context, sqldb *sql.DB, factory ctxn.Factory) (trillian.TrillianMapClient, error) {
-	tree, err := sqlhist.New(ctx, *mapID, factory)
-	if err != nil {
-		return nil, fmt.Errorf("sqlhist.New(): %v", err)
-	}
-
-	sths, err := sequenced.New(sqldb, *mapID)
-	if err != nil {
-		return nil, err
-	}
-	signer, err := keys.NewFromPrivatePEMFile(*signingKey, *signingKeyPassword)
-	if err != nil {
-		return nil, err
-	}
-
-	return mapserver.New(*mapID, tree, factory, sths, signer,
-		util.SystemTimeSource{}), nil
-}
-
 func main() {
 	flag.Parse()
 
@@ -108,20 +82,11 @@ func main() {
 	factory := transaction.NewFactory(sqldb)
 
 	// Connect to map server.
-	var tmap trillian.TrillianMapClient
-	if *mapURL != "" {
-		mconn, err := grpc.Dial(*mapURL, grpc.WithInsecure())
-		if err != nil {
-			glog.Exitf("grpc.Dial(%v): %v", *mapURL, err)
-		}
-		tmap = trillian.NewTrillianMapClient(mconn)
-	} else {
-		var err error
-		tmap, err = newMapServer(context.Background(), sqldb, factory)
-		if err != nil {
-			glog.Exitf("newMapServer: %v", err)
-		}
+	mconn, err := grpc.Dial(*mapURL, grpc.WithInsecure())
+	if err != nil {
+		glog.Exitf("grpc.Dial(%v): %v", *mapURL, err)
 	}
+	tmap := trillian.NewTrillianMapClient(mconn)
 
 	// Connection to append only log
 	tlog, err := config.LogClient(*logID, *logURL, *logPubKey)

core/client/kt/verify.go

@@ -24,14 +24,14 @@ import (
 
 	"github.com/google/keytransparency/core/crypto/commitments"
 	"github.com/google/keytransparency/core/crypto/vrf"
-	"github.com/google/keytransparency/core/tree/sparse"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/google/trillian"
 	"github.com/google/trillian/client"
+	"github.com/google/trillian/merkle"
+	"github.com/google/trillian/merkle/hashers"
 	"golang.org/x/net/context"
 
-	tv "github.com/google/keytransparency/core/tree/sparse/verifier"
 	tcrypto "github.com/google/trillian/crypto"
 
 	tpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types"
@@ -47,22 +47,22 @@ var (
 
 // Verifier is a client helper library for verifying request and responses.
 type Verifier struct {
-	vrf  vrf.PublicKey
-	tree *tv.Verifier
-	sig  crypto.PublicKey
-	log  client.LogVerifier
+	vrf    vrf.PublicKey
+	hasher hashers.MapHasher
+	sig    crypto.PublicKey
+	log    client.LogVerifier
 }
 
 // New creates a new instance of the client verifier.
 func New(vrf vrf.PublicKey,
-	tree *tv.Verifier,
+	hasher hashers.MapHasher,
 	sig crypto.PublicKey,
 	log client.LogVerifier) *Verifier {
 	return &Verifier{
-		vrf:  vrf,
-		tree: tree,
-		sig:  sig,
-		log:  log,
+		vrf:    vrf,
+		hasher: hasher,
+		sig:    sig,
+		log:    log,
 	}
 }
 
@@ -107,10 +107,14 @@ func (v *Verifier) VerifyGetEntryResponse(ctx context.Context, userID, appID str
 		return ErrNilProof
 	}
 
+	leaf := leafProof.GetLeaf().GetLeafValue()
+	proof := leafProof.GetInclusion()
+	expectedRoot := in.GetSmr().GetRootHash()
 	mapID := in.GetSmr().GetMapId()
-	if err := v.tree.VerifyProof(mapID, leafProof.Inclusion, index[:], leafProof.Leaf.LeafValue, sparse.FromBytes(in.GetSmr().RootHash)); err != nil {
+	leafHash := v.hasher.HashLeaf(mapID, index[:], leaf)
+	if err := merkle.VerifyMapInclusionProof(mapID, index[:], leafHash, expectedRoot, proof, v.hasher); err != nil {
 		Vlog.Printf("✗ Sparse tree proof verification failed.")
-		return fmt.Errorf("tree.VerifyProof(): %v", err)
+		return fmt.Errorf("VerifyMapInclusionProof(): %v", err)
 	}
 	Vlog.Printf("✓ Sparse tree proof verified.")