From f15da06af7aae354c30958602f2afafbe1fa874e Mon Sep 17 00:00:00 2001 From: "J.C. Jones" Date: Wed, 27 May 2015 12:06:33 -0700 Subject: [PATCH 1/3] Issue #238 - MySql column width too narrow - Added SQL debug logging (SA option: "SQLDebug") - Added timestamps to the log prints to stdout - Ignore *.pem in test/js - Modified start.sh to support environment overrides for BOULDER_CONFIG, like the AMQP mode - Changed boulder-test-config to open the server on the loopback device, so as to not cause firewall prompts on each integration test run for those of us being restrictive - Renamed "key" column to "jwk" in DB, to avoid keyword conflict - Set MaxLength on "jwk" column to 512 --- .gitignore | 1 + cmd/boulder-sa/main.go | 1 + cmd/boulder/main.go | 1 + cmd/shell.go | 1 + core/objects.go | 2 +- log/audit-logger.go | 3 ++- sa/storage-authority.go | 30 +++++++++++++++++++++++++++--- start.sh | 5 ++++- test/boulder-config.json | 3 ++- test/boulder-test-config.json | 5 +++-- 10 files changed, 43 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 36266b9ae2b..3ca9a00c3da 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ bin # Test files test/js/node_modules +test/js/*.pem # Architecture specific extensions/prefixes *.[568vq] diff --git a/cmd/boulder-sa/main.go b/cmd/boulder-sa/main.go index ebe970491d6..aed735605d4 100644 --- a/cmd/boulder-sa/main.go +++ b/cmd/boulder-sa/main.go @@ -35,6 +35,7 @@ func main() { sai, err := sa.NewSQLStorageAuthority(c.SA.DBDriver, c.SA.DBName) cmd.FailOnError(err, "Failed to create SA impl") + sai.SetSQLDebug(c.SA.SQLDebug) go cmd.ProfileCmd("SA", stats) diff --git a/cmd/boulder/main.go b/cmd/boulder/main.go index fd0085c7367..3f9b0efb010 100644 --- a/cmd/boulder/main.go +++ b/cmd/boulder/main.go @@ -77,6 +77,7 @@ func main() { wfe := wfe.NewWebFrontEndImpl() sa, err := sa.NewSQLStorageAuthority(c.SA.DBDriver, c.SA.DBName) cmd.FailOnError(err, "Unable to create SA") + sa.SetSQLDebug(c.SA.SQLDebug) ra := ra.NewRegistrationAuthorityImpl() va := va.NewValidationAuthorityImpl(c.CA.TestMode) diff --git a/cmd/shell.go b/cmd/shell.go index 614d109421d..6ededbe2743 100644 --- a/cmd/shell.go +++ b/cmd/shell.go @@ -67,6 +67,7 @@ type Config struct { SA struct { DBDriver string DBName string + SQLDebug bool } Statsd struct { diff --git a/core/objects.go b/core/objects.go index b936a1f43cf..836d0c58f47 100644 --- a/core/objects.go +++ b/core/objects.go @@ -103,7 +103,7 @@ type Registration struct { ID int64 `json:"-" db:"id"` // Account key to which the details are attached - Key jose.JsonWebKey `json:"key" db:"key"` + Key jose.JsonWebKey `json:"key" db:"jwk"` // Recovery Token is used to prove connection to an earlier transaction RecoveryToken string `json:"recoveryToken" db:"recoveryToken"` diff --git a/log/audit-logger.go b/log/audit-logger.go index 0edcad03c6a..aa3082fd754 100644 --- a/log/audit-logger.go +++ b/log/audit-logger.go @@ -12,6 +12,7 @@ import ( "log/syslog" "os" "sync" + "time" "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd" ) @@ -117,7 +118,7 @@ func GetAuditLogger() *AuditLogger { // Log the provided message at the appropriate level, writing to // both stdout and the Logger, as well as informing statsd. func (log *AuditLogger) logAtLevel(level, msg string) (err error) { - fmt.Printf("%s\n", msg) + fmt.Printf("%s %s\n", time.Now().Format("2006/01/02 15:04:05"), msg) log.Stats.Inc(level, 1, 1.0) switch level { diff --git a/sa/storage-authority.go b/sa/storage-authority.go index 045850eb284..a70e3ba4fd3 100644 --- a/sa/storage-authority.go +++ b/sa/storage-authority.go @@ -159,6 +159,17 @@ func NewDbMap(driver, dbName string) (dbMap *gorp.DbMap, err error) { return } +// SQLLogger adapts the AuditLogger to a format GORP can use. +type SQLLogger struct { + log *blog.AuditLogger +} + +// Printf adapts the AuditLogger to GORP's interface +func (log *SQLLogger) Printf(format string, v ...interface{}) { + log.log.Debug(fmt.Sprintf(format, v)) +} + +// NewSQLStorageAuthority provides persistence using a SQL backend for Boulder. func NewSQLStorageAuthority(driver string, name string) (ssa *SQLStorageAuthority, err error) { logger := blog.GetAuditLogger() logger.Notice("Storage Authority Starting") @@ -182,8 +193,21 @@ func NewSQLStorageAuthority(driver string, name string) (ssa *SQLStorageAuthorit return } +// SetSQLDebug enables/disables GORP SQL-level Debugging +func (ssa *SQLStorageAuthority) SetSQLDebug(state bool) { + ssa.dbMap.TraceOff() + + if state { + // Enable logging + ssa.dbMap.TraceOn("SQL: ", &SQLLogger{blog.GetAuditLogger()}) + } +} + func (ssa *SQLStorageAuthority) InitTables() (err error) { - ssa.dbMap.AddTableWithName(core.Registration{}, "registrations").SetKeys(true, "ID").SetVersionCol("LockCol") + regTable := ssa.dbMap.AddTableWithName(core.Registration{}, "registrations").SetKeys(true, "ID") + regTable.SetVersionCol("LockCol") + regTable.ColMap("Key").SetMaxSize(512).SetNotNull(true) + ssa.dbMap.AddTableWithName(pendingauthzModel{}, "pending_authz").SetKeys(false, "ID").SetVersionCol("LockCol") ssa.dbMap.AddTableWithName(authzModel{}, "authz").SetKeys(false, "ID") ssa.dbMap.AddTableWithName(core.Certificate{}, "certificates").SetKeys(false, "Serial") @@ -207,7 +231,7 @@ func (ssa *SQLStorageAuthority) DumpTables() error { fmt.Printf("\n----- registrations -----\n") var registrations []core.Registration - _, err = tx.Select(®istrations, "SELECT * FROM registrations ") + _, err = tx.Select(®istrations, "SELECT * FROM registrations") if err != nil { tx.Rollback() return err @@ -338,7 +362,7 @@ func (ssa *SQLStorageAuthority) GetRegistrationByKey(key jose.JsonWebKey) (reg c return } - err = ssa.dbMap.SelectOne(®, "SELECT * FROM registrations WHERE key = :key", map[string]interface{}{"key": string(keyJson)}) + err = ssa.dbMap.SelectOne(®, "SELECT * FROM registrations WHERE jwk = :key", map[string]interface{}{"key": string(keyJson)}) return } diff --git a/start.sh b/start.sh index 2e634492f42..2512b76f61b 100755 --- a/start.sh +++ b/start.sh @@ -6,7 +6,10 @@ fi # Kill all children on exit. trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT -go run ./cmd/boulder/main.go --config test/boulder-config.json & + +BOULDER_CONFIG=${BOULDER_CONFIG:-test/boulder-config.json} + +go run ./cmd/boulder/main.go & go run Godeps/_workspace/src/github.com/cloudflare/cfssl/cmd/cfssl/cfssl.go \ -loglevel 0 \ serve \ diff --git a/test/boulder-config.json b/test/boulder-config.json index f43b087444f..d659fb5250f 100644 --- a/test/boulder-config.json +++ b/test/boulder-config.json @@ -51,7 +51,8 @@ "sa": { "dbDriver": "sqlite3", - "dbName": ":memory:" + "dbName": ":memory:", + "SQLDebug": false }, "revoker": { diff --git a/test/boulder-test-config.json b/test/boulder-test-config.json index 0ebaeb61634..5e52230aed8 100644 --- a/test/boulder-test-config.json +++ b/test/boulder-test-config.json @@ -32,7 +32,7 @@ "wfe": { "baseURL": "http://localhost:4300", - "listenAddress": "0.0.0.0:4300" + "listenAddress": "127.0.0.1:4300" }, "ca": { @@ -51,7 +51,8 @@ "sa": { "dbDriver": "sqlite3", - "dbName": ":memory:" + "dbName": ":memory:", + "SQLDebug": false }, "mail": { From 343920cfe385a25063a17cf1a7636a4fe0bc7d54 Mon Sep 17 00:00:00 2001 From: "J.C. Jones" Date: Wed, 27 May 2015 13:36:04 -0700 Subject: [PATCH 2/3] Fix integration test while running with MySQL - Add SQL configuration options - Increase the width of the authz and pending_authz tables' challenges field - Make it configurable whether CREATE TABLE commands should run --- ca/certificate-authority-data.go | 6 ++---- ca/certificate-authority-data_test.go | 9 +++++++++ ca/certificate-authority_test.go | 6 +++++- cmd/boulder-ca/main.go | 5 +++++ cmd/boulder-sa/main.go | 7 ++++++- cmd/boulder/main.go | 10 +++++++++- cmd/shell.go | 6 +++++- core/interfaces.go | 2 +- ra/registration-authority_test.go | 6 +++++- sa/storage-authority.go | 22 ++++++++++++++-------- sa/storage-authority_test.go | 2 +- test/boulder-config.json | 8 ++++++-- test/boulder-test-config.json | 8 ++++++-- 13 files changed, 74 insertions(+), 23 deletions(-) diff --git a/ca/certificate-authority-data.go b/ca/certificate-authority-data.go index 33d0c17f946..0ab8645b775 100644 --- a/ca/certificate-authority-data.go +++ b/ca/certificate-authority-data.go @@ -39,16 +39,14 @@ func NewCertificateAuthorityDatabaseImpl(driver string, name string) (cadb core. db: db, log: logger, } - - err = createTablesIfNotExist(db) return } // createTablesIfNotExist builds the database tables and inserts the initial // state, if the tables do not already exist. It is not an error for the tables // to already exist. -func createTablesIfNotExist(db *sql.DB) (err error) { - tx, err := db.Begin() +func (cadb *CertificateAuthorityDatabaseImpl) CreateTablesIfNotExists() (err error) { + tx, err := cadb.db.Begin() if err != nil { return } diff --git a/ca/certificate-authority-data_test.go b/ca/certificate-authority-data_test.go index 39f13acc4b1..d883239120f 100644 --- a/ca/certificate-authority-data_test.go +++ b/ca/certificate-authority-data_test.go @@ -35,6 +35,9 @@ func TestBeginCommit(t *testing.T) { cadb, err := NewCertificateAuthorityDatabaseImpl(sqliteDriver, sqliteName) test.AssertNotError(t, err, "Could not construct CA DB") + err = cadb.CreateTablesIfNotExists() + test.AssertNotError(t, err, "Could not construct tables") + err = cadb.Begin() test.AssertNotError(t, err, "Could not begin") @@ -53,6 +56,9 @@ func TestGetSetSequenceOutsideTx(t *testing.T) { cadb, err := NewCertificateAuthorityDatabaseImpl(sqliteDriver, sqliteName) test.AssertNotError(t, err, "Could not construct CA DB") + err = cadb.CreateTablesIfNotExists() + test.AssertNotError(t, err, "Could not construct tables") + _, err = cadb.IncrementAndGetSerial() test.AssertError(t, err, "Not permitted") } @@ -61,6 +67,9 @@ func TestGetSetSequenceNumber(t *testing.T) { cadb, err := NewCertificateAuthorityDatabaseImpl(sqliteDriver, sqliteName) test.AssertNotError(t, err, "Could not construct CA DB") + err = cadb.CreateTablesIfNotExists() + test.AssertNotError(t, err, "Could not construct tables") + err = cadb.Begin() test.AssertNotError(t, err, "Could not begin") diff --git a/ca/certificate-authority_test.go b/ca/certificate-authority_test.go index 0bf0ab1a9b7..2afeef5a07b 100644 --- a/ca/certificate-authority_test.go +++ b/ca/certificate-authority_test.go @@ -322,11 +322,15 @@ func (cadb *MockCADatabase) IncrementAndGetSerial() (int, error) { return 1, nil } +func (cadb *MockCADatabase) CreateTablesIfNotExists() error { + return nil +} + func setup(t *testing.T) (cadb core.CertificateAuthorityDatabase, storageAuthority core.StorageAuthority, caConfig Config) { // Create an SA ssa, err := sa.NewSQLStorageAuthority("sqlite3", ":memory:") test.AssertNotError(t, err, "Failed to create SA") - ssa.InitTables() + ssa.CreateTablesIfNotExists() storageAuthority = ssa cadb, _ = NewMockCertificateAuthorityDatabase() diff --git a/cmd/boulder-ca/main.go b/cmd/boulder-ca/main.go index bb3ba69e348..d9d7ac1f409 100644 --- a/cmd/boulder-ca/main.go +++ b/cmd/boulder-ca/main.go @@ -36,6 +36,11 @@ func main() { cadb, err := ca.NewCertificateAuthorityDatabaseImpl(c.CA.DBDriver, c.CA.DBName) cmd.FailOnError(err, "Failed to create CA database") + if c.SQL.CreateTables { + err = cadb.CreateTablesIfNotExists() + cmd.FailOnError(err, "Failed to create CA tables") + } + cai, err := ca.NewCertificateAuthorityImpl(cadb, c.CA) cmd.FailOnError(err, "Failed to create CA impl") diff --git a/cmd/boulder-sa/main.go b/cmd/boulder-sa/main.go index aed735605d4..4276d55e4fe 100644 --- a/cmd/boulder-sa/main.go +++ b/cmd/boulder-sa/main.go @@ -35,7 +35,12 @@ func main() { sai, err := sa.NewSQLStorageAuthority(c.SA.DBDriver, c.SA.DBName) cmd.FailOnError(err, "Failed to create SA impl") - sai.SetSQLDebug(c.SA.SQLDebug) + sai.SetSQLDebug(c.SQL.SQLDebug) + + if c.SQL.CreateTables { + err = sai.CreateTablesIfNotExists() + cmd.FailOnError(err, "Failed to create tables") + } go cmd.ProfileCmd("SA", stats) diff --git a/cmd/boulder/main.go b/cmd/boulder/main.go index 3f9b0efb010..e47d8c4cffe 100644 --- a/cmd/boulder/main.go +++ b/cmd/boulder/main.go @@ -77,7 +77,7 @@ func main() { wfe := wfe.NewWebFrontEndImpl() sa, err := sa.NewSQLStorageAuthority(c.SA.DBDriver, c.SA.DBName) cmd.FailOnError(err, "Unable to create SA") - sa.SetSQLDebug(c.SA.SQLDebug) + sa.SetSQLDebug(c.SQL.SQLDebug) ra := ra.NewRegistrationAuthorityImpl() va := va.NewValidationAuthorityImpl(c.CA.TestMode) @@ -88,6 +88,14 @@ func main() { ca, err := ca.NewCertificateAuthorityImpl(cadb, c.CA) cmd.FailOnError(err, "Unable to create CA") + if c.SQL.CreateTables { + err = sa.CreateTablesIfNotExists() + cmd.FailOnError(err, "Failed to create SA tables") + + err = cadb.CreateTablesIfNotExists() + cmd.FailOnError(err, "Failed to create CA tables") + } + // Wire them up wfe.RA = &ra wfe.SA = sa diff --git a/cmd/shell.go b/cmd/shell.go index 6ededbe2743..bc38fa9e838 100644 --- a/cmd/shell.go +++ b/cmd/shell.go @@ -67,7 +67,11 @@ type Config struct { SA struct { DBDriver string DBName string - SQLDebug bool + } + + SQL struct { + CreateTables bool + SQLDebug bool } Statsd struct { diff --git a/core/interfaces.go b/core/interfaces.go index 548920040b0..106671b631a 100644 --- a/core/interfaces.go +++ b/core/interfaces.go @@ -119,9 +119,9 @@ type StorageAuthority interface { // CertificateAuthorityDatabase represents an atomic sequence source type CertificateAuthorityDatabase interface { + CreateTablesIfNotExists() error Begin() error Commit() error Rollback() error - IncrementAndGetSerial() (int, error) } diff --git a/ra/registration-authority_test.go b/ra/registration-authority_test.go index 4a635f9fe27..c4d976bc3b8 100644 --- a/ra/registration-authority_test.go +++ b/ra/registration-authority_test.go @@ -62,6 +62,10 @@ func (cadb *MockCADatabase) IncrementAndGetSerial() (int, error) { return 1, nil } +func (cadb *MockCADatabase) CreateTablesIfNotExists() error { + return nil +} + var ( // These values we simulate from the client AccountKeyJSON = []byte(`{ @@ -116,7 +120,7 @@ func initAuthorities(t *testing.T) (core.CertificateAuthority, *DummyValidationA sa, err := sa.NewSQLStorageAuthority("sqlite3", ":memory:") test.AssertNotError(t, err, "Failed to create SA") - sa.InitTables() + sa.CreateTablesIfNotExists() va := &DummyValidationAuthority{} diff --git a/sa/storage-authority.go b/sa/storage-authority.go index a70e3ba4fd3..a0bd5e0bb22 100644 --- a/sa/storage-authority.go +++ b/sa/storage-authority.go @@ -185,11 +185,7 @@ func NewSQLStorageAuthority(driver string, name string) (ssa *SQLStorageAuthorit bucket: make(map[string]interface{}), } - err = ssa.InitTables() - if err != nil { - return - } - + ssa.initTables() return } @@ -203,19 +199,29 @@ func (ssa *SQLStorageAuthority) SetSQLDebug(state bool) { } } -func (ssa *SQLStorageAuthority) InitTables() (err error) { +// initTables constructs the table map for the ORM. If you want to also create +// the tables, call CreateTablesIfNotExists. +func (ssa *SQLStorageAuthority) initTables() { regTable := ssa.dbMap.AddTableWithName(core.Registration{}, "registrations").SetKeys(true, "ID") regTable.SetVersionCol("LockCol") regTable.ColMap("Key").SetMaxSize(512).SetNotNull(true) - ssa.dbMap.AddTableWithName(pendingauthzModel{}, "pending_authz").SetKeys(false, "ID").SetVersionCol("LockCol") - ssa.dbMap.AddTableWithName(authzModel{}, "authz").SetKeys(false, "ID") + pendingAuthzTable := ssa.dbMap.AddTableWithName(pendingauthzModel{}, "pending_authz").SetKeys(false, "ID") + pendingAuthzTable.SetVersionCol("LockCol") + pendingAuthzTable.ColMap("Challenges").SetMaxSize(1536) + + authzTable := ssa.dbMap.AddTableWithName(authzModel{}, "authz").SetKeys(false, "ID") + authzTable.ColMap("Challenges").SetMaxSize(1536) + ssa.dbMap.AddTableWithName(core.Certificate{}, "certificates").SetKeys(false, "Serial") ssa.dbMap.AddTableWithName(core.CertificateStatus{}, "certificateStatus").SetKeys(false, "Serial").SetVersionCol("LockCol") ssa.dbMap.AddTableWithName(core.OcspResponse{}, "ocspResponses").SetKeys(true, "ID") ssa.dbMap.AddTableWithName(core.Crl{}, "crls").SetKeys(false, "Serial") ssa.dbMap.AddTableWithName(core.DeniedCsr{}, "deniedCsrs").SetKeys(true, "ID") +} +// CreateTablesIfNotExists instructs the ORM to create any missing tables. +func (ssa *SQLStorageAuthority) CreateTablesIfNotExists() (err error) { err = ssa.dbMap.CreateTablesIfNotExists() return } diff --git a/sa/storage-authority_test.go b/sa/storage-authority_test.go index 35a820b9570..1f31e80bf0f 100644 --- a/sa/storage-authority_test.go +++ b/sa/storage-authority_test.go @@ -28,7 +28,7 @@ func initSA(t *testing.T) *SQLStorageAuthority { if err != nil { t.Fatalf("Failed to create SA") } - if err = sa.InitTables(); err != nil { + if err = sa.CreateTablesIfNotExists(); err != nil { t.Fatalf("Failed to create SA") } return sa diff --git a/test/boulder-config.json b/test/boulder-config.json index d659fb5250f..6b6a63197ad 100644 --- a/test/boulder-config.json +++ b/test/boulder-config.json @@ -51,8 +51,12 @@ "sa": { "dbDriver": "sqlite3", - "dbName": ":memory:", - "SQLDebug": false + "dbName": ":memory:" + }, + + "sql": { + "SQLDebug": true, + "CreateTables": false }, "revoker": { diff --git a/test/boulder-test-config.json b/test/boulder-test-config.json index 5e52230aed8..188ce031f34 100644 --- a/test/boulder-test-config.json +++ b/test/boulder-test-config.json @@ -51,8 +51,12 @@ "sa": { "dbDriver": "sqlite3", - "dbName": ":memory:", - "SQLDebug": false + "dbName": ":memory:" + }, + + "sql": { + "SQLDebug": false, + "CreateTables": true }, "mail": { From 714432e9d79b782d6184b1985e4c0717998842aa Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 27 May 2015 16:08:04 -0700 Subject: [PATCH 3/3] Max JWK size of 1024. --- sa/storage-authority.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sa/storage-authority.go b/sa/storage-authority.go index a0bd5e0bb22..73febc79959 100644 --- a/sa/storage-authority.go +++ b/sa/storage-authority.go @@ -204,7 +204,7 @@ func (ssa *SQLStorageAuthority) SetSQLDebug(state bool) { func (ssa *SQLStorageAuthority) initTables() { regTable := ssa.dbMap.AddTableWithName(core.Registration{}, "registrations").SetKeys(true, "ID") regTable.SetVersionCol("LockCol") - regTable.ColMap("Key").SetMaxSize(512).SetNotNull(true) + regTable.ColMap("Key").SetMaxSize(1024).SetNotNull(true) pendingAuthzTable := ssa.dbMap.AddTableWithName(pendingauthzModel{}, "pending_authz").SetKeys(false, "ID") pendingAuthzTable.SetVersionCol("LockCol")