Add Canary TLS support (uber#5086)

* add support for TLS connections by Canary, add development config for Canary with TLS

* update README to include new config option

* remove testing config

---------

Co-authored-by: David Porter <david.porter@uber.com>
Co-authored-by: Shijie Sheng <shengs@uber.com>
Co-authored-by: Zijian <Shaddoll@users.noreply.github.com>

canary/README.md

@@ -84,6 +84,7 @@ cadence:
   service: "cadence-frontend" # frontend service name
   address: "127.0.0.1:7833" # frontend address
   #host: "127.0.0.1:7933" # replace address with host if using Thrift for compatibility
+  #tlsCaFile: "path/to/file" # give file path to TLS CA file if TLS is enabled on the Cadence server
   #metrics: ... # optional detailed client side metrics like workflow latency. But for monitoring, simply use server side metrics `workflow_success` is enough.
 ```
 - **Metrics**: metrics configuration. Similar to server metric emitter, only M3/Statsd/Prometheus is supported. 

canary/config.go

@@ -87,6 +87,8 @@ type (
 		ThriftHostNameAndPort string `yaml:"host"`
 		// gRPC host name and port
 		GRPCHostNameAndPort string `yaml:"address"`
+		// TLS cert file if TLS is enabled on the Cadence server
+		TLSCAFile string `yaml:"tlsCaFile"`
 	}
 )
 

canary/runner.go

@@ -25,12 +25,20 @@ import (
 	"sync"
 	"time"
 
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
+
 	"go.uber.org/cadence/.gen/go/cadence/workflowserviceclient"
 	"go.uber.org/cadence/compatibility"
 	"go.uber.org/yarpc"
+	"go.uber.org/yarpc/api/transport"
+	"go.uber.org/yarpc/peer"
+	"go.uber.org/yarpc/peer/hostport"
 	"go.uber.org/yarpc/transport/grpc"
 	"go.uber.org/yarpc/transport/tchannel"
 	"go.uber.org/zap"
+	"google.golang.org/grpc/credentials"
 
 	apiv1 "github.com/uber/cadence-idl/go/proto/api/v1"
 
@@ -59,10 +67,30 @@ func NewCanaryRunner(cfg *Config) (Runnable, error) {
 	var dispatcher *yarpc.Dispatcher
 	var runtimeContext *RuntimeContext
 	if cfg.Cadence.GRPCHostNameAndPort != "" {
+		var outbounds transport.Outbounds
+		if cfg.Cadence.TLSCAFile != "" {
+			caCert, err := ioutil.ReadFile(cfg.Cadence.TLSCAFile)
+			if err != nil {
+				logger.Fatal("Failed to load server CA certificate", zap.Error(err))
+			}
+			caCertPool := x509.NewCertPool()
+			if !caCertPool.AppendCertsFromPEM(caCert) {
+				logger.Fatal("Failed to add server CA certificate", zap.Error(err))
+			}
+			tlsConfig := tls.Config{
+				RootCAs: caCertPool,
+			}
+			tlsCreds := credentials.NewTLS(&tlsConfig)
+			grpcTransport := grpc.NewTransport()
+			tlsChooser := peer.NewSingle(hostport.Identify(cfg.Cadence.GRPCHostNameAndPort), grpcTransport.NewDialer(grpc.DialerCredentials(tlsCreds)))
+			outbounds = transport.Outbounds{Unary: grpcTransport.NewOutbound(tlsChooser)}
+		} else {
+			outbounds = transport.Outbounds{Unary: grpc.NewTransport().NewSingleOutbound(cfg.Cadence.GRPCHostNameAndPort)}
+		}
 		dispatcher = yarpc.NewDispatcher(yarpc.Config{
 			Name: CanaryServiceName,
 			Outbounds: yarpc.Outbounds{
-				cfg.Cadence.ServiceName: {Unary: grpc.NewTransport().NewSingleOutbound(cfg.Cadence.GRPCHostNameAndPort)},
+				cfg.Cadence.ServiceName: outbounds,
 			},
 		})
 		clientConfig := dispatcher.ClientConfig(cfg.Cadence.ServiceName)