Skip to content

Commit 1656c95

Browse files
gyuhogyuho
committed
integration: test TLS reload
Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>
1 parent 4e21f87 commit 1656c95

File tree

2 files changed

+263
-0
lines changed

2 files changed

+263
-0
lines changed

‎integration/cluster.go

+48
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,14 @@ package integration
1717
import (
1818
"crypto/tls"
1919
"fmt"
20+
"io"
2021
"io/ioutil"
2122
"math/rand"
2223
"net"
2324
"net/http"
2425
"net/http/httptest"
2526
"os"
27+
"path/filepath"
2628
"reflect"
2729
"sort"
2830
"strings"
@@ -76,6 +78,13 @@ var (
7678
ClientCertAuth: true,
7779
}
7880

81+
testTLSInfoExpired = transport.TLSInfo{
82+
KeyFile: "./fixtures-expired/server-key.pem",
83+
CertFile: "./fixtures-expired/server.pem",
84+
TrustedCAFile: "./fixtures-expired/etcd-root-ca.pem",
85+
ClientCertAuth: true,
86+
}
87+
7988
plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "integration")
8089
)
8190

@@ -929,3 +938,42 @@ type grpcAPI struct {
929938
// Auth is the authentication API for the client's connection.
930939
Auth pb.AuthClient
931940
}
941+
942+
// copyTLSFiles clones certs files to dst directory.
943+
func copyTLSFiles(ti transport.TLSInfo, dst string) (transport.TLSInfo, error) {
944+
ci := transport.TLSInfo{
945+
KeyFile: filepath.Join(dst, "server-key.pem"),
946+
CertFile: filepath.Join(dst, "server.pem"),
947+
TrustedCAFile: filepath.Join(dst, "etcd-root-ca.pem"),
948+
ClientCertAuth: ti.ClientCertAuth,
949+
}
950+
if err := copyFile(ti.KeyFile, ci.KeyFile); err != nil {
951+
return transport.TLSInfo{}, err
952+
}
953+
if err := copyFile(ti.CertFile, ci.CertFile); err != nil {
954+
return transport.TLSInfo{}, err
955+
}
956+
if err := copyFile(ti.TrustedCAFile, ci.TrustedCAFile); err != nil {
957+
return transport.TLSInfo{}, err
958+
}
959+
return ci, nil
960+
}
961+
962+
func copyFile(src, dst string) error {
963+
f, err := os.Open(src)
964+
if err != nil {
965+
return err
966+
}
967+
defer f.Close()
968+
969+
w, err := os.Create(dst)
970+
if err != nil {
971+
return err
972+
}
973+
defer w.Close()
974+
975+
if _, err = io.Copy(w, f); err != nil {
976+
return err
977+
}
978+
return w.Sync()
979+
}

‎integration/v3_grpc_test.go

+215
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,22 @@ package integration
1616

1717
import (
1818
"bytes"
19+
"crypto/tls"
1920
"fmt"
21+
"io/ioutil"
2022
"math/rand"
2123
"os"
2224
"reflect"
25+
"strings"
2326
"testing"
2427
"time"
2528

29+
"github.com/coreos/etcd/clientv3"
2630
"github.com/coreos/etcd/etcdserver/api/v3rpc"
2731
"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
2832
pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
2933
"github.com/coreos/etcd/pkg/testutil"
34+
3035
"golang.org/x/net/context"
3136
"google.golang.org/grpc"
3237
"google.golang.org/grpc/metadata"
@@ -1374,6 +1379,216 @@ func TestTLSGRPCAcceptSecureAll(t *testing.T) {
13741379
}
13751380
}
13761381

1382+
// TestTLSReloadAtomicReplace ensures server reloads expired/valid certs
1383+
// when all certs are atomically replaced by directory renaming.
1384+
// And expects server to reject client requests, and vice versa.
1385+
func TestTLSReloadAtomicReplace(t *testing.T) {
1386+
defer testutil.AfterTest(t)
1387+
1388+
// clone valid,expired certs to separate directories for atomic renaming
1389+
vDir, err := ioutil.TempDir(os.TempDir(), "fixtures-valid")
1390+
if err != nil {
1391+
t.Fatal(err)
1392+
}
1393+
defer os.RemoveAll(vDir)
1394+
ts, err := copyTLSFiles(testTLSInfo, vDir)
1395+
if err != nil {
1396+
t.Fatal(err)
1397+
}
1398+
eDir, err := ioutil.TempDir(os.TempDir(), "fixtures-expired")
1399+
if err != nil {
1400+
t.Fatal(err)
1401+
}
1402+
defer os.RemoveAll(eDir)
1403+
if _, err = copyTLSFiles(testTLSInfoExpired, eDir); err != nil {
1404+
t.Fatal(err)
1405+
}
1406+
1407+
tDir, err := ioutil.TempDir(os.TempDir(), "fixtures")
1408+
if err != nil {
1409+
t.Fatal(err)
1410+
}
1411+
os.RemoveAll(tDir)
1412+
defer os.RemoveAll(tDir)
1413+
1414+
// start with valid certs
1415+
clus := NewClusterV3(t, &ClusterConfig{Size: 1, PeerTLS: &ts, ClientTLS: &ts})
1416+
defer clus.Terminate(t)
1417+
1418+
// concurrent client dialing while certs transition from valid to expired
1419+
errc := make(chan error)
1420+
go func() {
1421+
for {
1422+
cc, err := ts.ClientConfig()
1423+
if err != nil {
1424+
if os.IsNotExist(err) {
1425+
// from concurrent renaming
1426+
continue
1427+
}
1428+
t.Fatal(err)
1429+
}
1430+
cli, cerr := clientv3.New(clientv3.Config{
1431+
Endpoints: []string{clus.Members[0].GRPCAddr()},
1432+
DialTimeout: 3 * time.Second,
1433+
TLS: cc,
1434+
})
1435+
if cerr != nil {
1436+
errc <- cerr
1437+
return
1438+
}
1439+
cli.Close()
1440+
}
1441+
}()
1442+
1443+
// replace certs directory with expired ones
1444+
if err = os.Rename(vDir, tDir); err != nil {
1445+
t.Fatal(err)
1446+
}
1447+
if err = os.Rename(eDir, vDir); err != nil {
1448+
t.Fatal(err)
1449+
}
1450+
1451+
// after rename,
1452+
// 'vDir' contains expired certs
1453+
// 'tDir' contains valid certs
1454+
// 'eDir' does not exist
1455+
1456+
select {
1457+
case err = <-errc:
1458+
if err != grpc.ErrClientConnTimeout {
1459+
t.Fatalf("expected %v, got %v", grpc.ErrClientConnTimeout, err)
1460+
}
1461+
case <-time.After(7 * time.Second):
1462+
t.Fatal("expected dial timeout in 3 seconds, but never got it")
1463+
}
1464+
1465+
// now, replace expired certs back with valid ones
1466+
if err = os.Rename(tDir, eDir); err != nil {
1467+
t.Fatal(err)
1468+
}
1469+
if err = os.Rename(vDir, tDir); err != nil {
1470+
t.Fatal(err)
1471+
}
1472+
if err = os.Rename(eDir, vDir); err != nil {
1473+
t.Fatal(err)
1474+
}
1475+
1476+
// new incoming client request should trigger
1477+
// listener to reload valid certs
1478+
var tls *tls.Config
1479+
tls, err = ts.ClientConfig()
1480+
if err != nil {
1481+
t.Fatal(err)
1482+
}
1483+
var cl *clientv3.Client
1484+
cl, err = clientv3.New(clientv3.Config{
1485+
Endpoints: []string{clus.Members[0].GRPCAddr()},
1486+
DialTimeout: 3 * time.Second,
1487+
TLS: tls,
1488+
})
1489+
if err != nil {
1490+
t.Fatalf("expected no error, got %v", err)
1491+
}
1492+
cl.Close()
1493+
}
1494+
1495+
// TestTLSReloadCopy ensures server reloads expired/valid certs
1496+
// when new certs are copied over, one by one. And expects server
1497+
// to reject client requests, and vice versa.
1498+
func TestTLSReloadCopy(t *testing.T) {
1499+
defer testutil.AfterTest(t)
1500+
1501+
// clone certs directory, free to overwrite
1502+
cDir, err := ioutil.TempDir(os.TempDir(), "fixtures-test")
1503+
if err != nil {
1504+
t.Fatal(err)
1505+
}
1506+
defer os.RemoveAll(cDir)
1507+
ts, err := copyTLSFiles(testTLSInfo, cDir)
1508+
if err != nil {
1509+
t.Fatal(err)
1510+
}
1511+
1512+
// start with valid certs
1513+
clus := NewClusterV3(t, &ClusterConfig{Size: 1, PeerTLS: &ts, ClientTLS: &ts})
1514+
defer clus.Terminate(t)
1515+
1516+
// concurrent client dialing while certs transition from valid to expired
1517+
errc := make(chan error)
1518+
go func() {
1519+
for {
1520+
cc, err := ts.ClientConfig()
1521+
if err != nil {
1522+
if strings.Contains(err.Error(), "tls: private key does not match public key") {
1523+
// from concurrent certs overwriting
1524+
continue
1525+
}
1526+
t.Fatal(err)
1527+
}
1528+
cli, cerr := clientv3.New(clientv3.Config{
1529+
Endpoints: []string{clus.Members[0].GRPCAddr()},
1530+
DialTimeout: 3 * time.Second,
1531+
TLS: cc,
1532+
})
1533+
if cerr != nil {
1534+
errc <- cerr
1535+
return
1536+
}
1537+
cli.Close()
1538+
}
1539+
}()
1540+
1541+
// overwrite valid certs with expired ones
1542+
// (e.g. simulate cert expiration in practice)
1543+
if err = copyFile(testTLSInfoExpired.KeyFile, ts.KeyFile); err != nil {
1544+
t.Fatal(err)
1545+
}
1546+
if err = copyFile(testTLSInfoExpired.CertFile, ts.CertFile); err != nil {
1547+
t.Fatal(err)
1548+
}
1549+
if err = copyFile(testTLSInfoExpired.TrustedCAFile, ts.TrustedCAFile); err != nil {
1550+
t.Fatal(err)
1551+
}
1552+
1553+
select {
1554+
case err = <-errc:
1555+
if err != grpc.ErrClientConnTimeout {
1556+
t.Fatalf("expected %v, got %v", grpc.ErrClientConnTimeout, err)
1557+
}
1558+
case <-time.After(7 * time.Second):
1559+
t.Fatal("expected dial timeout in 3 seconds, but never got it")
1560+
}
1561+
1562+
// now, replace expired certs back with valid ones
1563+
if err = copyFile(testTLSInfo.KeyFile, ts.KeyFile); err != nil {
1564+
t.Fatal(err)
1565+
}
1566+
if err = copyFile(testTLSInfo.CertFile, ts.CertFile); err != nil {
1567+
t.Fatal(err)
1568+
}
1569+
if err = copyFile(testTLSInfo.TrustedCAFile, ts.TrustedCAFile); err != nil {
1570+
t.Fatal(err)
1571+
}
1572+
1573+
// new incoming client request should trigger
1574+
// listener to reload valid certs
1575+
var tls *tls.Config
1576+
tls, err = ts.ClientConfig()
1577+
if err != nil {
1578+
t.Fatal(err)
1579+
}
1580+
var cl *clientv3.Client
1581+
cl, err = clientv3.New(clientv3.Config{
1582+
Endpoints: []string{clus.Members[0].GRPCAddr()},
1583+
DialTimeout: 3 * time.Second,
1584+
TLS: tls,
1585+
})
1586+
if err != nil {
1587+
t.Fatalf("expected no error, got %v", err)
1588+
}
1589+
cl.Close()
1590+
}
1591+
13771592
func TestGRPCRequireLeader(t *testing.T) {
13781593
defer testutil.AfterTest(t)
13791594

0 commit comments

Comments
 (0)