BREAKING CHANGE: fix: pin certificates to hosts (#40)

Before writing this diff, netem automatically generated the correct cert
on the fly based on the given SNI or local addr.

While this behavior has been okay so far, it turns out there is a set of
tests we cannot write because of it.

Namely, we cannot check whether we can connect to a given host using
another SNI, because netem would generate a certificate for the provided
SNI, which is not how the internet works.

If I'm connecting to www.example.com with www.google.com as the SNI, in
the internet the server would return a valid certificate for
www.example.com, rather than for www.google.com.

This diff rectifies netem's behavior by forcing the programmer to pin
the certificate to a set of names and addresses when creating the server
that needs such a certificate.

Accordingly, we can stop using a forked google/martian/v3/mitm
implementation and we can just roll out our own, which is still based on
martian (fear not, not reinventing the wheel here!) but allows us to
create a certificate with specific addresses and domain names pinned to
it.

While there, notice that there was code we were not using, such as
stdlib.go, and that we also don't need in ooni/probe-cli, so we can
definitely kill this piece of code.

While there, don't be shy and make a bunch of constructors of the `Must`
kind. They will panic on failure, which is fine in the netem context,
because this library is meant to only be used when writing tests.

Part of ooni/probe#2531, because the tests I
was trying to write belong to such an issue.

ca.go

@@ -0,0 +1,210 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package netem
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha1"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"math/big"
+	"net"
+	"time"
+)
+
+// caMaxSerialNumber is the upper boundary that is used to create unique serial
+// numbers for the certificate. This can be any unsigned integer up to 20
+// bytes (2^(8*20)-1).
+var caMaxSerialNumber = big.NewInt(0).SetBytes(bytes.Repeat([]byte{255}, 20))
+
+// caMustNewAuthority creates a new CA certificate and associated private key or PANICS.
+//
+// This code is derived from github.com/google/martian/v3.
+//
+// SPDX-License-Identifier: Apache-2.0.
+func caMustNewAuthority(name, organization string, validity time.Duration,
+	timeNow func() time.Time) (*x509.Certificate, *rsa.PrivateKey) {
+	priv := Must1(rsa.GenerateKey(rand.Reader, 2048))
+	pub := priv.Public()
+
+	// Subject Key Identifier support for end entity certificate.
+	// https://www.ietf.org/rfc/rfc3280.txt (section 4.2.1.2)
+	pkixpub := Must1(x509.MarshalPKIXPublicKey(pub))
+	h := sha1.New()
+	h.Write(pkixpub)
+	keyID := h.Sum(nil)
+
+	// TODO(bassosimone): keep a map of used serial numbers to avoid potentially
+	// reusing a serial multiple times.
+	serial := Must1(rand.Int(rand.Reader, caMaxSerialNumber))
+
+	tmpl := &x509.Certificate{
+		SerialNumber: serial,
+		Subject: pkix.Name{
+			CommonName:   name,
+			Organization: []string{organization},
+		},
+		SubjectKeyId:          keyID,
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		NotBefore:             timeNow().Add(-validity),
+		NotAfter:              timeNow().Add(validity),
+		DNSNames:              []string{name},
+		IsCA:                  true,
+	}
+
+	raw := Must1(x509.CreateCertificate(rand.Reader, tmpl, tmpl, pub, priv))
+
+	// Parse certificate bytes so that we have a leaf certificate.
+	x509c := Must1(x509.ParseCertificate(raw))
+
+	return x509c, priv
+}
+
+// CA is a certification authority.
+//
+// The zero value is invalid, please use [NewCA] to construct.
+//
+// This code is derived from github.com/google/martian/v3.
+//
+// SPDX-License-Identifier: Apache-2.0.
+type CA struct {
+	ca       *x509.Certificate
+	capriv   any
+	keyID    []byte
+	org      string
+	priv     *rsa.PrivateKey
+	validity time.Duration
+}
+
+// NewCA creates a new certification authority.
+func MustNewCA() *CA {
+	return MustNewCAWithTimeNow(time.Now)
+}
+
+// MustNewCA is like [NewCA] but uses a custom [time.Now] func.
+//
+// This code is derived from github.com/google/martian/v3.
+//
+// SPDX-License-Identifier: Apache-2.0.
+func MustNewCAWithTimeNow(timeNow func() time.Time) *CA {
+	ca, privateKey := caMustNewAuthority("jafar", "OONI", 24*time.Hour, timeNow)
+
+	roots := x509.NewCertPool()
+	roots.AddCert(ca)
+
+	priv := Must1(rsa.GenerateKey(rand.Reader, 2048))
+	pub := priv.Public()
+
+	// Subject Key Identifier support for end entity certificate.
+	// https://www.ietf.org/rfc/rfc3280.txt (section 4.2.1.2)
+	pkixpub := Must1(x509.MarshalPKIXPublicKey(pub))
+	h := sha1.New()
+	h.Write(pkixpub)
+	keyID := h.Sum(nil)
+
+	return &CA{
+		ca:       ca,
+		capriv:   privateKey,
+		priv:     priv,
+		keyID:    keyID,
+		validity: time.Hour,
+		org:      "OONI Netem CA",
+	}
+}
+
+// CertPool returns an [x509.CertPool] using the given [*CA].
+func (c *CA) CertPool() *x509.CertPool {
+	pool := x509.NewCertPool()
+	pool.AddCert(c.ca)
+	return pool
+}
+
+// MustNewCert creates a new certificate for the given common name or PANICS.
+//
+// The common name and the extra names could contain domain names or IP addresses.
+//
+// For example:
+//
+// - www.example.com
+//
+// - 10.0.0.1
+//
+// - ::1
+//
+// are all valid values you can pass as common name or extra names.
+func (c *CA) MustNewCert(commonName string, extraNames ...string) *tls.Certificate {
+	return c.MustNewCertWithTimeNow(time.Now, commonName, extraNames...)
+}
+
+// MustNewCertWithTimeNow is like [MustNewCert] but uses a custom [time.Now] func.
+//
+// This code is derived from github.com/google/martian/v3.
+//
+// SPDX-License-Identifier: Apache-2.0.
+func (c *CA) MustNewCertWithTimeNow(timeNow func() time.Time, commonName string, extraNames ...string) *tls.Certificate {
+	serial := Must1(rand.Int(rand.Reader, caMaxSerialNumber))
+
+	tmpl := &x509.Certificate{
+		SerialNumber: serial,
+		Subject: pkix.Name{
+			CommonName:   commonName,
+			Organization: []string{c.org},
+		},
+		SubjectKeyId:          c.keyID,
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		NotBefore:             timeNow().Add(-c.validity),
+		NotAfter:              timeNow().Add(c.validity),
+	}
+
+	allNames := []string{commonName}
+	allNames = append(allNames, extraNames...)
+	for _, name := range allNames {
+		if ip := net.ParseIP(name); ip != nil {
+			tmpl.IPAddresses = append(tmpl.IPAddresses, ip)
+		} else {
+			tmpl.DNSNames = append(tmpl.DNSNames, name)
+		}
+	}
+
+	raw := Must1(x509.CreateCertificate(rand.Reader, tmpl, c.ca, c.priv.Public(), c.capriv))
+
+	// Parse certificate bytes so that we have a leaf certificate.
+	x509c := Must1(x509.ParseCertificate(raw))
+
+	tlsc := &tls.Certificate{
+		Certificate: [][]byte{raw, c.ca.Raw},
+		PrivateKey:  c.priv,
+		Leaf:        x509c,
+	}
+
+	return tlsc
+}
+
+// MustServerTLSConfig generates a server-side [*tls.Config] that uses the given [*CA] and
+// a generated certificate for the given common name and extra names.
+//
+// See [CA.MustNewCert] documentation for more details about what common name and extra names should be.
+func (ca *CA) MustServerTLSConfig(commonName string, extraNames ...string) *tls.Config {
+	return &tls.Config{
+		Certificates: []tls.Certificate{*ca.MustNewCert(commonName, extraNames...)},
+	}
+}

ca_test.go

@@ -0,0 +1,142 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package netem
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"net"
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestCAMustNewCert(t *testing.T) {
+	ca := MustNewCA()
+
+	tlsc := ca.MustNewCert("example.com", "www.example.com", "10.0.0.1", "10.0.0.2")
+
+	if tlsc.Certificate == nil {
+		t.Error("tlsc.Certificate: got nil, want certificate bytes")
+	}
+	if tlsc.PrivateKey == nil {
+		t.Error("tlsc.PrivateKey: got nil, want private key")
+	}
+
+	x509c := tlsc.Leaf
+	if x509c == nil {
+		t.Fatal("x509c: got nil, want *x509.Certificate")
+	}
+
+	if got := x509c.SerialNumber; got.Cmp(caMaxSerialNumber) >= 0 {
+		t.Errorf("x509c.SerialNumber: got %v, want <= MaxSerialNumber", got)
+	}
+	if got, want := x509c.Subject.CommonName, "example.com"; got != want {
+		t.Errorf("X509c.Subject.CommonName: got %q, want %q", got, want)
+	}
+	if err := x509c.VerifyHostname("example.com"); err != nil {
+		t.Errorf("x509c.VerifyHostname(%q): got %v, want no error", "example.com", err)
+	}
+
+	if got, want := x509c.Subject.Organization, []string{"OONI Netem CA"}; !reflect.DeepEqual(got, want) {
+		t.Errorf("x509c.Subject.Organization: got %v, want %v", got, want)
+	}
+
+	if got := x509c.SubjectKeyId; got == nil {
+		t.Error("x509c.SubjectKeyId: got nothing, want key ID")
+	}
+	if !x509c.BasicConstraintsValid {
+		t.Error("x509c.BasicConstraintsValid: got false, want true")
+	}
+
+	if got, want := x509c.KeyUsage, x509.KeyUsageKeyEncipherment; got&want == 0 {
+		t.Error("x509c.KeyUsage: got nothing, want to include x509.KeyUsageKeyEncipherment")
+	}
+	if got, want := x509c.KeyUsage, x509.KeyUsageDigitalSignature; got&want == 0 {
+		t.Error("x509c.KeyUsage: got nothing, want to include x509.KeyUsageDigitalSignature")
+	}
+
+	want := []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}
+	if got := x509c.ExtKeyUsage; !reflect.DeepEqual(got, want) {
+		t.Errorf("x509c.ExtKeyUsage: got %v, want %v", got, want)
+	}
+
+	if got, want := x509c.DNSNames, []string{"example.com", "www.example.com"}; !reflect.DeepEqual(got, want) {
+		t.Errorf("x509c.DNSNames: got %v, want %v", got, want)
+	}
+
+	if diff := cmp.Diff(x509c.IPAddresses, []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("10.0.0.2")}); diff != "" {
+		t.Errorf(diff)
+	}
+
+	before := time.Now().Add(-2 * time.Hour)
+	if got := x509c.NotBefore; before.After(got) {
+		t.Errorf("x509c.NotBefore: got %v, want after %v", got, before)
+	}
+
+	after := time.Now().Add(2 * time.Hour)
+	if got := x509c.NotAfter; !after.After(got) {
+		t.Errorf("x509c.NotAfter: got %v, want before %v", got, want)
+	}
+}
+
+func TestCAWeCanGenerateAnExpiredCertificate(t *testing.T) {
+	topology := MustNewStarTopology(log.Log)
+	defer topology.Close()
+
+	serverStack := Must1(topology.AddHost("10.0.0.1", "0.0.0.0", &LinkConfig{}))
+	clientStack := Must1(topology.AddHost("10.0.0.2", "0.0.0.0", &LinkConfig{}))
+
+	serverAddr := &net.TCPAddr{IP: net.IPv4(10, 0, 0, 1), Port: 443}
+	serverListener := Must1(serverStack.ListenTCP("tcp", serverAddr))
+
+	serverServer := &http.Server{
+		Handler: http.NewServeMux(),
+		TLSConfig: &tls.Config{
+			Certificates: []tls.Certificate{
+				*serverStack.CA().MustNewCertWithTimeNow(func() time.Time {
+					return time.Date(2017, time.July, 17, 0, 0, 0, 0, time.UTC)
+				},
+					"www.example.com",
+					"10.0.0.1",
+				),
+			},
+		},
+	}
+	go serverServer.ServeTLS(serverListener, "", "")
+	defer serverServer.Close()
+
+	tcpConn, err := clientStack.DialContext(context.Background(), "tcp", "10.0.0.1:443")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tcpConn.Close()
+
+	tlsClientConfig := &tls.Config{
+		RootCAs:    clientStack.DefaultCertPool(),
+		ServerName: "www.example.com",
+	}
+	tlsConn := tls.Client(tcpConn, tlsClientConfig)
+	err = tlsConn.HandshakeContext(context.Background())
+	if err == nil || !strings.Contains(err.Error(), "x509: certificate has expired or is not yet valid") {
+		t.Fatal("unexpected error", err)
+	}
+}

cmd/calibrate/topology.go

@@ -64,7 +64,7 @@ func newTopologyStar(
 	dnsConfig *netem.DNSConfig,
 ) (topologyCloser, *netem.UNetStack, *netem.UNetStack) {
 	// create an empty topology
-	topology := netem.Must1(netem.NewStarTopology(log.Log))
+	topology := netem.MustNewStarTopology(log.Log)
 
 	// add the client to the topology
 	clientStack := netem.Must1(topology.AddHost(clientAddress, serverAddress, clientLink))
@@ -102,12 +102,12 @@ func newTopologyPPP(
 	dnsConfig *netem.DNSConfig,
 ) (topologyCloser, *netem.UNetStack, *netem.UNetStack) {
 	// create a PPP topology
-	topology := netem.Must1(netem.NewPPPTopology(
+	topology := netem.MustNewPPPTopology(
 		clientAddress,
 		serverAddress,
 		log.Log,
 		clientLink,
-	))
+	)
 
 	// create DNS server using the server stack
 	_ = netem.Must1(netem.NewDNSServer(log.Log, topology.Server, serverAddress, dnsConfig))