Merge pull request #241 from letsencrypt/238-key_index

Issue #238 - MySql column width too narrow

.gitignore

@@ -10,6 +10,7 @@ bin
 
 # Test files
 test/js/node_modules
+test/js/*.pem
 
 # Architecture specific extensions/prefixes
 *.[568vq]

ca/certificate-authority-data.go

@@ -39,16 +39,14 @@ func NewCertificateAuthorityDatabaseImpl(driver string, name string) (cadb core.
 		db:  db,
 		log: logger,
 	}
-
-	err = createTablesIfNotExist(db)
 	return
 }
 
 // createTablesIfNotExist builds the database tables and inserts the initial
 // state, if the tables do not already exist. It is not an error for the tables
 // to already exist.
-func createTablesIfNotExist(db *sql.DB) (err error) {
-	tx, err := db.Begin()
+func (cadb *CertificateAuthorityDatabaseImpl) CreateTablesIfNotExists() (err error) {
+	tx, err := cadb.db.Begin()
 	if err != nil {
 		return
 	}

ca/certificate-authority-data_test.go

@@ -35,6 +35,9 @@ func TestBeginCommit(t *testing.T) {
 	cadb, err := NewCertificateAuthorityDatabaseImpl(sqliteDriver, sqliteName)
 	test.AssertNotError(t, err, "Could not construct CA DB")
 
+	err = cadb.CreateTablesIfNotExists()
+	test.AssertNotError(t, err, "Could not construct tables")
+
 	err = cadb.Begin()
 	test.AssertNotError(t, err, "Could not begin")
 
@@ -53,6 +56,9 @@ func TestGetSetSequenceOutsideTx(t *testing.T) {
 	cadb, err := NewCertificateAuthorityDatabaseImpl(sqliteDriver, sqliteName)
 	test.AssertNotError(t, err, "Could not construct CA DB")
 
+	err = cadb.CreateTablesIfNotExists()
+	test.AssertNotError(t, err, "Could not construct tables")
+
 	_, err = cadb.IncrementAndGetSerial()
 	test.AssertError(t, err, "Not permitted")
 }
@@ -61,6 +67,9 @@ func TestGetSetSequenceNumber(t *testing.T) {
 	cadb, err := NewCertificateAuthorityDatabaseImpl(sqliteDriver, sqliteName)
 	test.AssertNotError(t, err, "Could not construct CA DB")
 
+	err = cadb.CreateTablesIfNotExists()
+	test.AssertNotError(t, err, "Could not construct tables")
+
 	err = cadb.Begin()
 	test.AssertNotError(t, err, "Could not begin")
 

ca/certificate-authority_test.go

@@ -322,11 +322,15 @@ func (cadb *MockCADatabase) IncrementAndGetSerial() (int, error) {
 	return 1, nil
 }
 
+func (cadb *MockCADatabase) CreateTablesIfNotExists() error {
+	return nil
+}
+
 func setup(t *testing.T) (cadb core.CertificateAuthorityDatabase, storageAuthority core.StorageAuthority, caConfig Config) {
 	// Create an SA
 	ssa, err := sa.NewSQLStorageAuthority("sqlite3", ":memory:")
 	test.AssertNotError(t, err, "Failed to create SA")
-	ssa.InitTables()
+	ssa.CreateTablesIfNotExists()
 	storageAuthority = ssa
 
 	cadb, _ = NewMockCertificateAuthorityDatabase()

cmd/boulder-ca/main.go

@@ -36,6 +36,11 @@ func main() {
 		cadb, err := ca.NewCertificateAuthorityDatabaseImpl(c.CA.DBDriver, c.CA.DBName)
 		cmd.FailOnError(err, "Failed to create CA database")
 
+		if c.SQL.CreateTables {
+			err = cadb.CreateTablesIfNotExists()
+			cmd.FailOnError(err, "Failed to create CA tables")
+		}
+
 		cai, err := ca.NewCertificateAuthorityImpl(cadb, c.CA)
 		cmd.FailOnError(err, "Failed to create CA impl")
 

cmd/boulder-sa/main.go

@@ -35,6 +35,12 @@ func main() {
 
 		sai, err := sa.NewSQLStorageAuthority(c.SA.DBDriver, c.SA.DBName)
 		cmd.FailOnError(err, "Failed to create SA impl")
+		sai.SetSQLDebug(c.SQL.SQLDebug)
+
+		if c.SQL.CreateTables {
+			err = sai.CreateTablesIfNotExists()
+			cmd.FailOnError(err, "Failed to create tables")
+		}
 
 		go cmd.ProfileCmd("SA", stats)
 

cmd/boulder/main.go

@@ -77,6 +77,7 @@ func main() {
 		wfe := wfe.NewWebFrontEndImpl()
 		sa, err := sa.NewSQLStorageAuthority(c.SA.DBDriver, c.SA.DBName)
 		cmd.FailOnError(err, "Unable to create SA")
+		sa.SetSQLDebug(c.SQL.SQLDebug)
 
 		ra := ra.NewRegistrationAuthorityImpl()
 		va := va.NewValidationAuthorityImpl(c.CA.TestMode)
@@ -87,6 +88,14 @@ func main() {
 		ca, err := ca.NewCertificateAuthorityImpl(cadb, c.CA)
 		cmd.FailOnError(err, "Unable to create CA")
 
+		if c.SQL.CreateTables {
+			err = sa.CreateTablesIfNotExists()
+			cmd.FailOnError(err, "Failed to create SA tables")
+
+			err = cadb.CreateTablesIfNotExists()
+			cmd.FailOnError(err, "Failed to create CA tables")
+		}
+
 		// Wire them up
 		wfe.RA = &ra
 		wfe.SA = sa

cmd/shell.go

@@ -69,6 +69,11 @@ type Config struct {
 		DBName   string
 	}
 
+	SQL struct {
+		CreateTables bool
+		SQLDebug     bool
+	}
+
 	Statsd struct {
 		Server string
 		Prefix string

core/interfaces.go

@@ -119,9 +119,9 @@ type StorageAuthority interface {
 
 // CertificateAuthorityDatabase represents an atomic sequence source
 type CertificateAuthorityDatabase interface {
+	CreateTablesIfNotExists() error
 	Begin() error
 	Commit() error
 	Rollback() error
-
 	IncrementAndGetSerial() (int, error)
 }

core/objects.go

@@ -103,7 +103,7 @@ type Registration struct {
 	ID int64 `json:"-" db:"id"`
 
 	// Account key to which the details are attached
-	Key jose.JsonWebKey `json:"key" db:"key"`
+	Key jose.JsonWebKey `json:"key" db:"jwk"`
 
 	// Recovery Token is used to prove connection to an earlier transaction
 	RecoveryToken string `json:"recoveryToken" db:"recoveryToken"`

log/audit-logger.go

@@ -12,6 +12,7 @@ import (
 	"log/syslog"
 	"os"
 	"sync"
+	"time"
 
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
 )
@@ -117,7 +118,7 @@ func GetAuditLogger() *AuditLogger {
 // Log the provided message at the appropriate level, writing to
 // both stdout and the Logger, as well as informing statsd.
 func (log *AuditLogger) logAtLevel(level, msg string) (err error) {
-	fmt.Printf("%s\n", msg)
+	fmt.Printf("%s %s\n", time.Now().Format("2006/01/02 15:04:05"), msg)
 	log.Stats.Inc(level, 1, 1.0)
 
 	switch level {

ra/registration-authority_test.go

@@ -62,6 +62,10 @@ func (cadb *MockCADatabase) IncrementAndGetSerial() (int, error) {
 	return 1, nil
 }
 
+func (cadb *MockCADatabase) CreateTablesIfNotExists() error {
+	return nil
+}
+
 var (
 	// These values we simulate from the client
 	AccountKeyJSON = []byte(`{
@@ -116,7 +120,7 @@ func initAuthorities(t *testing.T) (core.CertificateAuthority, *DummyValidationA
 
 	sa, err := sa.NewSQLStorageAuthority("sqlite3", ":memory:")
 	test.AssertNotError(t, err, "Failed to create SA")
-	sa.InitTables()
+	sa.CreateTablesIfNotExists()
 
 	va := &DummyValidationAuthority{}
 

sa/storage-authority.go

@@ -159,6 +159,17 @@ func NewDbMap(driver, dbName string) (dbMap *gorp.DbMap, err error) {
 	return
 }
 
+// SQLLogger adapts the AuditLogger to a format GORP can use.
+type SQLLogger struct {
+	log *blog.AuditLogger
+}
+
+// Printf adapts the AuditLogger to GORP's interface
+func (log *SQLLogger) Printf(format string, v ...interface{}) {
+	log.log.Debug(fmt.Sprintf(format, v))
+}
+
+// NewSQLStorageAuthority provides persistence using a SQL backend for Boulder.
 func NewSQLStorageAuthority(driver string, name string) (ssa *SQLStorageAuthority, err error) {
 	logger := blog.GetAuditLogger()
 	logger.Notice("Storage Authority Starting")
@@ -174,24 +185,43 @@ func NewSQLStorageAuthority(driver string, name string) (ssa *SQLStorageAuthorit
 		bucket: make(map[string]interface{}),
 	}
 
-	err = ssa.InitTables()
-	if err != nil {
-		return
-	}
-
+	ssa.initTables()
 	return
 }
 
-func (ssa *SQLStorageAuthority) InitTables() (err error) {
-	ssa.dbMap.AddTableWithName(core.Registration{}, "registrations").SetKeys(true, "ID").SetVersionCol("LockCol")
-	ssa.dbMap.AddTableWithName(pendingauthzModel{}, "pending_authz").SetKeys(false, "ID").SetVersionCol("LockCol")
-	ssa.dbMap.AddTableWithName(authzModel{}, "authz").SetKeys(false, "ID")
+// SetSQLDebug enables/disables GORP SQL-level Debugging
+func (ssa *SQLStorageAuthority) SetSQLDebug(state bool) {
+	ssa.dbMap.TraceOff()
+
+	if state {
+		// Enable logging
+		ssa.dbMap.TraceOn("SQL: ", &SQLLogger{blog.GetAuditLogger()})
+	}
+}
+
+// initTables constructs the table map for the ORM. If you want to also create
+// the tables, call CreateTablesIfNotExists.
+func (ssa *SQLStorageAuthority) initTables() {
+	regTable := ssa.dbMap.AddTableWithName(core.Registration{}, "registrations").SetKeys(true, "ID")
+	regTable.SetVersionCol("LockCol")
+	regTable.ColMap("Key").SetMaxSize(1024).SetNotNull(true)
+
+	pendingAuthzTable := ssa.dbMap.AddTableWithName(pendingauthzModel{}, "pending_authz").SetKeys(false, "ID")
+	pendingAuthzTable.SetVersionCol("LockCol")
+	pendingAuthzTable.ColMap("Challenges").SetMaxSize(1536)
+
+	authzTable := ssa.dbMap.AddTableWithName(authzModel{}, "authz").SetKeys(false, "ID")
+	authzTable.ColMap("Challenges").SetMaxSize(1536)
+
 	ssa.dbMap.AddTableWithName(core.Certificate{}, "certificates").SetKeys(false, "Serial")
 	ssa.dbMap.AddTableWithName(core.CertificateStatus{}, "certificateStatus").SetKeys(false, "Serial").SetVersionCol("LockCol")
 	ssa.dbMap.AddTableWithName(core.OcspResponse{}, "ocspResponses").SetKeys(true, "ID")
 	ssa.dbMap.AddTableWithName(core.Crl{}, "crls").SetKeys(false, "Serial")
 	ssa.dbMap.AddTableWithName(core.DeniedCsr{}, "deniedCsrs").SetKeys(true, "ID")
+}
 
+// CreateTablesIfNotExists instructs the ORM to create any missing tables.
+func (ssa *SQLStorageAuthority) CreateTablesIfNotExists() (err error) {
 	err = ssa.dbMap.CreateTablesIfNotExists()
 	return
 }
@@ -207,7 +237,7 @@ func (ssa *SQLStorageAuthority) DumpTables() error {
 
 	fmt.Printf("\n----- registrations -----\n")
 	var registrations []core.Registration
-	_, err = tx.Select(&registrations, "SELECT * FROM registrations ")
+	_, err = tx.Select(&registrations, "SELECT * FROM registrations")
 	if err != nil {
 		tx.Rollback()
 		return err
@@ -338,7 +368,7 @@ func (ssa *SQLStorageAuthority) GetRegistrationByKey(key jose.JsonWebKey) (reg c
 		return
 	}
 
-	err = ssa.dbMap.SelectOne(&reg, "SELECT * FROM registrations WHERE key = :key", map[string]interface{}{"key": string(keyJson)})
+	err = ssa.dbMap.SelectOne(&reg, "SELECT * FROM registrations WHERE jwk = :key", map[string]interface{}{"key": string(keyJson)})
 	return
 }
 

sa/storage-authority_test.go

@@ -28,7 +28,7 @@ func initSA(t *testing.T) *SQLStorageAuthority {
 	if err != nil {
 		t.Fatalf("Failed to create SA")
 	}
-	if err = sa.InitTables(); err != nil {
+	if err = sa.CreateTablesIfNotExists(); err != nil {
 		t.Fatalf("Failed to create SA")
 	}
 	return sa

start.sh

@@ -6,7 +6,10 @@ fi
 
 # Kill all children on exit.
 trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
-go run ./cmd/boulder/main.go --config test/boulder-config.json &
+
+BOULDER_CONFIG=${BOULDER_CONFIG:-test/boulder-config.json}
+
+go run ./cmd/boulder/main.go &
 go run Godeps/_workspace/src/github.com/cloudflare/cfssl/cmd/cfssl/cfssl.go \
   -loglevel 0 \
   serve \

test/boulder-config.json

@@ -54,6 +54,11 @@
     "dbName": ":memory:"
   },
 
+  "sql": {
+    "SQLDebug": true,
+    "CreateTables": false
+  },
+
   "revoker": {
     "dbDriver": "sqlite3",
     "dbName": ":memory:"

test/boulder-test-config.json

@@ -32,7 +32,7 @@
 
   "wfe": {
     "baseURL": "http://localhost:4300",
-    "listenAddress": "0.0.0.0:4300"
+    "listenAddress": "127.0.0.1:4300"
   },
 
   "ca": {
@@ -54,6 +54,11 @@
     "dbName": ":memory:"
   },
 
+  "sql": {
+    "SQLDebug": false,
+    "CreateTables": true
+  },
+
   "mail": {
     "server": "mail.example.com",
     "port": "25",