Skip to content

Commit c2f7574

Browse files
gyuhogyuho
committed
integration: test TLS reload
Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>
1 parent 9408f66 commit c2f7574

File tree

2 files changed

+266
-0
lines changed

2 files changed

+266
-0
lines changed

‎integration/cluster.go

+48
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,14 @@ package integration
1717
import (
1818
"crypto/tls"
1919
"fmt"
20+
"io"
2021
"io/ioutil"
2122
"math/rand"
2223
"net"
2324
"net/http"
2425
"net/http/httptest"
2526
"os"
27+
"path/filepath"
2628
"reflect"
2729
"sort"
2830
"strings"
@@ -76,6 +78,13 @@ var (
7678
ClientCertAuth: true,
7779
}
7880

81+
testTLSInfoExpired = transport.TLSInfo{
82+
KeyFile: "./fixtures-expired/server-key.pem",
83+
CertFile: "./fixtures-expired/server.pem",
84+
TrustedCAFile: "./fixtures-expired/etcd-root-ca.pem",
85+
ClientCertAuth: true,
86+
}
87+
7988
plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "integration")
8089
)
8190

@@ -929,3 +938,42 @@ type grpcAPI struct {
929938
// Auth is the authentication API for the client's connection.
930939
Auth pb.AuthClient
931940
}
941+
942+
// copyTLSFiles clones certs files to dst directory.
943+
func copyTLSFiles(ti transport.TLSInfo, dst string) (transport.TLSInfo, error) {
944+
ci := transport.TLSInfo{
945+
KeyFile: filepath.Join(dst, "server-key.pem"),
946+
CertFile: filepath.Join(dst, "server.pem"),
947+
TrustedCAFile: filepath.Join(dst, "etcd-root-ca.pem"),
948+
ClientCertAuth: ti.ClientCertAuth,
949+
}
950+
if err := copyFile(ti.KeyFile, ci.KeyFile); err != nil {
951+
return transport.TLSInfo{}, err
952+
}
953+
if err := copyFile(ti.CertFile, ci.CertFile); err != nil {
954+
return transport.TLSInfo{}, err
955+
}
956+
if err := copyFile(ti.TrustedCAFile, ci.TrustedCAFile); err != nil {
957+
return transport.TLSInfo{}, err
958+
}
959+
return ci, nil
960+
}
961+
962+
func copyFile(src, dst string) error {
963+
f, err := os.Open(src)
964+
if err != nil {
965+
return err
966+
}
967+
defer f.Close()
968+
969+
w, err := os.Create(dst)
970+
if err != nil {
971+
return err
972+
}
973+
defer w.Close()
974+
975+
if _, err = io.Copy(w, f); err != nil {
976+
return err
977+
}
978+
return w.Sync()
979+
}

‎integration/v3_grpc_test.go

+218
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,18 @@ package integration
1616

1717
import (
1818
"bytes"
19+
"crypto/tls"
1920
"fmt"
21+
"io/ioutil"
2022
"math/rand"
2123
"os"
2224
"reflect"
2325
"testing"
2426
"time"
2527

28+
"strings"
29+
30+
"github.com/coreos/etcd/clientv3"
2631
"github.com/coreos/etcd/etcdserver/api/v3rpc"
2732
"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
2833
pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
@@ -1374,6 +1379,219 @@ func TestTLSGRPCAcceptSecureAll(t *testing.T) {
13741379
}
13751380
}
13761381

1382+
// TestTLSReloadAtomicReplace ensures server reloads expired/valid certs
1383+
// when all certs are atomically replaced by directory renaming.
1384+
// And expects server to reject client requests, and vice versa.
1385+
func TestTLSReloadAtomicReplace(t *testing.T) {
1386+
defer testutil.AfterTest(t)
1387+
1388+
// clone valid,expired certs to separate directories for atomic renaming
1389+
vDir, err := ioutil.TempDir(os.TempDir(), "fixtures-valid")
1390+
if err != nil {
1391+
t.Fatal(err)
1392+
}
1393+
defer os.RemoveAll(vDir)
1394+
ts, err := copyTLSFiles(testTLSInfo, vDir)
1395+
if err != nil {
1396+
t.Fatal(err)
1397+
}
1398+
eDir, err := ioutil.TempDir(os.TempDir(), "fixtures-expired")
1399+
if err != nil {
1400+
t.Fatal(err)
1401+
}
1402+
defer os.RemoveAll(eDir)
1403+
if _, err = copyTLSFiles(testTLSInfoExpired, eDir); err != nil {
1404+
t.Fatal(err)
1405+
}
1406+
1407+
var tDir string
1408+
tDir, err = ioutil.TempDir(os.TempDir(), "fixtures")
1409+
if err != nil {
1410+
t.Fatal(err)
1411+
}
1412+
os.RemoveAll(tDir)
1413+
defer os.RemoveAll(tDir)
1414+
1415+
// start with valid certs
1416+
clus := NewClusterV3(t, &ClusterConfig{Size: 1, PeerTLS: &ts, ClientTLS: &ts})
1417+
defer clus.Terminate(t)
1418+
1419+
// concurrent client dialing while certs transition
1420+
// from valid to expired ones
1421+
errc := make(chan error)
1422+
go func() {
1423+
for {
1424+
cc, err := ts.ClientConfig()
1425+
if err != nil {
1426+
if os.IsNotExist(err) {
1427+
// from concurrent renaming
1428+
continue
1429+
}
1430+
t.Fatal(err)
1431+
}
1432+
cli, cerr := clientv3.New(clientv3.Config{
1433+
Endpoints: []string{clus.Members[0].GRPCAddr()},
1434+
DialTimeout: 3 * time.Second,
1435+
TLS: cc,
1436+
})
1437+
if cerr != nil {
1438+
errc <- cerr
1439+
return
1440+
}
1441+
cli.Close()
1442+
}
1443+
}()
1444+
1445+
// replace certs directory with expired ones
1446+
if err = os.Rename(vDir, tDir); err != nil {
1447+
t.Fatal(err)
1448+
}
1449+
if err = os.Rename(eDir, vDir); err != nil {
1450+
t.Fatal(err)
1451+
}
1452+
1453+
// after rename,
1454+
// 'vDir' contains expired certs
1455+
// 'tDir' contains valid certs
1456+
// 'eDir' does not exist
1457+
1458+
select {
1459+
case err = <-errc:
1460+
if err != grpc.ErrClientConnTimeout {
1461+
t.Fatalf("expected %v, got %v", grpc.ErrClientConnTimeout, err)
1462+
}
1463+
case <-time.After(7 * time.Second):
1464+
t.Fatal("expected dial timeout in 3 seconds, but never got it")
1465+
}
1466+
1467+
// now, replace expired certs back with valid ones
1468+
if err = os.Rename(tDir, eDir); err != nil {
1469+
t.Fatal(err)
1470+
}
1471+
if err = os.Rename(vDir, tDir); err != nil {
1472+
t.Fatal(err)
1473+
}
1474+
if err = os.Rename(eDir, vDir); err != nil {
1475+
t.Fatal(err)
1476+
}
1477+
1478+
// new incoming client request should trigger
1479+
// listener to reload valid certs
1480+
var tls *tls.Config
1481+
tls, err = ts.ClientConfig()
1482+
if err != nil {
1483+
t.Fatal(err)
1484+
}
1485+
var cl *clientv3.Client
1486+
cl, err = clientv3.New(clientv3.Config{
1487+
Endpoints: []string{clus.Members[0].GRPCAddr()},
1488+
DialTimeout: 3 * time.Second,
1489+
TLS: tls,
1490+
})
1491+
if err != nil {
1492+
t.Fatalf("expected no error, got %v", err)
1493+
}
1494+
cl.Close()
1495+
}
1496+
1497+
// TestTLSReloadCopy ensures server reloads expired/valid certs
1498+
// when new certs are copied over, one by one. And expects server
1499+
// to reject client requests, and vice versa.
1500+
func TestTLSReloadCopy(t *testing.T) {
1501+
defer testutil.AfterTest(t)
1502+
1503+
// clone certs directory, free to overwrite
1504+
cDir, err := ioutil.TempDir(os.TempDir(), "fixtures-test")
1505+
if err != nil {
1506+
t.Fatal(err)
1507+
}
1508+
defer os.RemoveAll(cDir)
1509+
ts, err := copyTLSFiles(testTLSInfo, cDir)
1510+
if err != nil {
1511+
t.Fatal(err)
1512+
}
1513+
1514+
// start with valid certs
1515+
clus := NewClusterV3(t, &ClusterConfig{Size: 1, PeerTLS: &ts, ClientTLS: &ts})
1516+
defer clus.Terminate(t)
1517+
1518+
// concurrent client dialing while certs transition
1519+
// from valid to expired ones
1520+
errc := make(chan error)
1521+
go func() {
1522+
for {
1523+
cc, err := ts.ClientConfig()
1524+
if err != nil {
1525+
if strings.Contains(err.Error(), "tls: private key does not match public key") {
1526+
// from concurrent certs overwriting
1527+
continue
1528+
}
1529+
t.Fatal(err)
1530+
}
1531+
cli, cerr := clientv3.New(clientv3.Config{
1532+
Endpoints: []string{clus.Members[0].GRPCAddr()},
1533+
DialTimeout: 3 * time.Second,
1534+
TLS: cc,
1535+
})
1536+
if cerr != nil {
1537+
errc <- cerr
1538+
return
1539+
}
1540+
cli.Close()
1541+
}
1542+
}()
1543+
1544+
// overwrite valid certs with expired ones
1545+
// (e.g. simulate cert expiration in practice)
1546+
if err = copyFile(testTLSInfoExpired.KeyFile, ts.KeyFile); err != nil {
1547+
t.Fatal(err)
1548+
}
1549+
if err = copyFile(testTLSInfoExpired.CertFile, ts.CertFile); err != nil {
1550+
t.Fatal(err)
1551+
}
1552+
if err = copyFile(testTLSInfoExpired.TrustedCAFile, ts.TrustedCAFile); err != nil {
1553+
t.Fatal(err)
1554+
}
1555+
1556+
select {
1557+
case err = <-errc:
1558+
if err != grpc.ErrClientConnTimeout {
1559+
t.Fatalf("expected %v, got %v", grpc.ErrClientConnTimeout, err)
1560+
}
1561+
case <-time.After(7 * time.Second):
1562+
t.Fatal("expected dial timeout in 3 seconds, but never got it")
1563+
}
1564+
1565+
// now, replace expired certs back with valid ones
1566+
if err = copyFile(testTLSInfo.KeyFile, ts.KeyFile); err != nil {
1567+
t.Fatal(err)
1568+
}
1569+
if err = copyFile(testTLSInfo.CertFile, ts.CertFile); err != nil {
1570+
t.Fatal(err)
1571+
}
1572+
if err = copyFile(testTLSInfo.TrustedCAFile, ts.TrustedCAFile); err != nil {
1573+
t.Fatal(err)
1574+
}
1575+
1576+
// new incoming client request should trigger
1577+
// listener to reload valid certs
1578+
var tls *tls.Config
1579+
tls, err = ts.ClientConfig()
1580+
if err != nil {
1581+
t.Fatal(err)
1582+
}
1583+
var cl *clientv3.Client
1584+
cl, err = clientv3.New(clientv3.Config{
1585+
Endpoints: []string{clus.Members[0].GRPCAddr()},
1586+
DialTimeout: 3 * time.Second,
1587+
TLS: tls,
1588+
})
1589+
if err != nil {
1590+
t.Fatalf("expected no error, got %v", err)
1591+
}
1592+
cl.Close()
1593+
}
1594+
13771595
func TestGRPCRequireLeader(t *testing.T) {
13781596
defer testutil.AfterTest(t)
13791597

0 commit comments

Comments
 (0)