refactor: move migrations to gobuffalo/fizz (#1775)

This patch deprecates the previous migration system (sql-migrate) in favor of gobuffalo/fizz. No functional changes have been made.

BREAKING CHANGES: Please run `hydra migrate sql` before applying this release.

.circleci/config.yml

@@ -10,19 +10,32 @@ orbs:
   golangci: ory/golangci@0.0.4
 
 jobs:
+  test-legacy-migrations:
+    parameters:
+      flavor:
+        type: string
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      - run: sudo rm -rfd /usr/local/go
+      - run: sudo circleci-install golang 1.14.2
+      - run: make test-resetdb
+      - run: source scripts/test-env.sh && go test -tags legacy_migration_test -failfast -timeout=20m -run=Test*/db=<< parameters.flavor>> ./internal/fizzmigrate
+
   test:
     docker:
       - image: circleci/golang:1.14
         environment:
           - GO111MODULE=on
-          - TEST_DATABASE_POSTGRESQL=postgres://test:test@localhost:5432/hydra?sslmode=disable
-          - TEST_DATABASE_MYSQL=mysql://root:test@(localhost:3306)/mysql?parseTime=true
+          - TEST_DATABASE_POSTGRESQL=postgres://test:test@localhost:5432/postgres?sslmode=disable
+          - TEST_DATABASE_MYSQL=mysql://root:test@(localhost:3306)/mysql?parseTime=true&multiStatements=true
           - TEST_DATABASE_COCKROACHDB=cockroach://root@localhost:26257/defaultdb?sslmode=disable
       - image: postgres:9.6
         environment:
           - POSTGRES_USER=test
           - POSTGRES_PASSWORD=test
-          - POSTGRES_DB=hydra
+          - POSTGRES_DB=postgres
       - image: mysql:5.7
         environment:
           - MYSQL_ROOT_PASSWORD=test
@@ -56,14 +69,14 @@ jobs:
       - image: oryd/e2e-env:latest
         environment:
           - GO111MODULE=on
-          - TEST_DATABASE_MYSQL=mysql://root:test@(localhost:3306)/mysql?parseTime=true
+          - TEST_DATABASE_MYSQL=mysql://root:test@(localhost:3306)/mysql?parseTime=true&multiStatements=true
           - TEST_DATABASE_COCKROACHDB=cockroach://root@localhost:26257/defaultdb?sslmode=disable
-          - TEST_DATABASE_POSTGRESQL=postgres://test:test@localhost:5432/hydra?sslmode=disable
+          - TEST_DATABASE_POSTGRESQL=postgres://test:test@localhost:5432/postgres?sslmode=disable
       - image: postgres:9.6
         environment:
           - POSTGRES_USER=test
           - POSTGRES_PASSWORD=test
-          - POSTGRES_DB=hydra
+          - POSTGRES_DB=postgres
       - image: cockroachdb/cockroach:v2.1.6
         command: start --insecure
       - image: mysql:5.7
@@ -115,6 +128,28 @@ jobs:
 workflows:
   bdt:
     jobs:
+
+      # Test legacy migrations
+      - test-legacy-migrations:
+          name: test-legacy-migrations-postgres
+          flavor: postgres
+          filters:
+            tags:
+              only: /.*/
+      - test-legacy-migrations:
+          name: test-legacy-migrations-mysql
+          flavor: mysql
+          filters:
+            tags:
+              only: /.*/
+      - test-legacy-migrations:
+          name: test-legacy-migrations-cockroach
+          flavor: cockroach
+          filters:
+            tags:
+              only: /.*/
+      # END - Test legacy migrations
+
       - golangci/lint:
           filters:
             tags:
@@ -168,6 +203,9 @@ workflows:
             - test-e2e-mysql
             - test-e2e-cockroach
             - test-e2e-plugin
+            - test-legacy-migrations-postgres
+            - test-legacy-migrations-mysql
+            - test-legacy-migrations-cockroach
           filters:
             tags:
               only: /.*/
@@ -184,6 +222,9 @@ workflows:
             - test-e2e-mysql
             - test-e2e-cockroach
             - test-e2e-plugin
+            - test-legacy-migrations-postgres
+            - test-legacy-migrations-mysql
+            - test-legacy-migrations-cockroach
           filters:
             tags:
               only: /.*/
@@ -228,6 +269,9 @@ workflows:
             - test-e2e-cockroach
             - test-e2e-plugin
             - changelog/generate
+            - test-legacy-migrations-postgres
+            - test-legacy-migrations-mysql
+            - test-legacy-migrations-cockroach
           filters:
             branches:
               ignore: /.*/

Makefile

@@ -5,15 +5,21 @@ tools:
 		npm i
 		go install github.com/ory/go-acc github.com/ory/x/tools/listx github.com/go-swagger/go-swagger/cmd/swagger github.com/go-bindata/go-bindata/go-bindata github.com/sqs/goreturns github.com/ory/sdk/swagutil
 
+# Runs full test suite including tests where databases are enabled
+.PHONY: test-legacy-migrations
+test-legacy-migrations:
+		make test-resetdb
+		make sqlbin
+		source scripts/test-env.sh && go test -tags legacy_migration_test -failfast -timeout=20m ./internal/fizzmigrate
+		docker rm -f hydra_test_database_mysql
+		docker rm -f hydra_test_database_postgres
+		docker rm -f hydra_test_database_cockroach
+
 # Runs full test suite including tests where databases are enabled
 .PHONY: test
 test:
 		make test-resetdb
-		make sqlbin
-		TEST_DATABASE_MYSQL='mysql://root:secret@(127.0.0.1:3444)/mysql?parseTime=true' \
-		TEST_DATABASE_POSTGRESQL='postgres://postgres:secret@127.0.0.1:3445/hydra?sslmode=disable' \
-		TEST_DATABASE_COCKROACHDB='cockroach://root@127.0.0.1:3446/defaultdb?sslmode=disable' \
-		$$(go env GOPATH)/bin/go-acc ./... -- -failfast -timeout=20m
+		source scripts/test-env.sh && $$(go env GOPATH)/bin/go-acc ./... -- -failfast -timeout=20m
 		docker rm -f hydra_test_database_mysql
 		docker rm -f hydra_test_database_postgres
 		docker rm -f hydra_test_database_cockroach
@@ -28,13 +34,12 @@ test-resetdb:
 		docker rm -f hydra_test_database_postgres || true
 		docker rm -f hydra_test_database_cockroach || true
 		docker run --rm --name hydra_test_database_mysql -p 3444:3306 -e MYSQL_ROOT_PASSWORD=secret -d mysql:5.7
-		docker run --rm --name hydra_test_database_postgres -p 3445:5432 -e POSTGRES_PASSWORD=secret -e POSTGRES_DB=hydra -d postgres:9.6
+		docker run --rm --name hydra_test_database_postgres -p 3445:5432 -e POSTGRES_PASSWORD=secret -e POSTGRES_DB=postgres -d postgres:9.6
 		docker run --rm --name hydra_test_database_cockroach -p 3446:26257 -d cockroachdb/cockroach:v2.1.6 start --insecure
 
 # Runs tests in short mode, without database adapters
 .PHONY: docker
 docker:
-		make sqlbin
 		packr2
 		CGO_ENABLED=0 GO111MODULE=on GOOS=linux GOARCH=amd64 go build
 		packr2 clean
@@ -76,10 +81,10 @@ mocks:
 # Adds sql files to the binary using go-bindata
 .PHONY: sqlbin
 sqlbin:
-		cd client; $$(go env GOPATH)/bin/go-bindata -o sql_migration_files.go -pkg client ./migrations/sql/...
-		cd consent; $$(go env GOPATH)/bin/go-bindata -o sql_migration_files.go -pkg consent ./migrations/sql/...
-		cd jwk; $$(go env GOPATH)/bin/go-bindata -o sql_migration_files.go -pkg jwk ./migrations/sql/...
-		cd oauth2; $$(go env GOPATH)/bin/go-bindata -o sql_migration_files.go -pkg oauth2 ./migrations/sql/...
+		cd internal/fizzmigrate/client; $$(go env GOPATH)/bin/go-bindata -o sql_migration_files.go -pkg client ./migrations/sql/...
+		cd internal/fizzmigrate/consent; $$(go env GOPATH)/bin/go-bindata -o sql_migration_files.go -pkg consent ./migrations/sql/...
+		cd internal/fizzmigrate/jwk; $$(go env GOPATH)/bin/go-bindata -o sql_migration_files.go -pkg jwk ./migrations/sql/...
+		cd internal/fizzmigrate/oauth2; $$(go env GOPATH)/bin/go-bindata -o sql_migration_files.go -pkg oauth2 ./migrations/sql/...
 
 # Runs all code generators
 .PHONY: gen

README.md

@@ -439,8 +439,8 @@ suggest that you initialize the databases with:
 
 ```shell script
 make resetdb
-export TEST_DATABASE_MYSQL='mysql://root:secret@(127.0.0.1:3444)/mysql?parseTime=true'
-export TEST_DATABASE_POSTGRESQL='postgres://postgres:secret@127.0.0.1:3445/hydra?sslmode=disable'
+export TEST_DATABASE_MYSQL='mysql://root:secret@(127.0.0.1:3444)/mysql?parseTime=true&multiStatements=true'
+export TEST_DATABASE_POSTGRESQL='postgres://postgres:secret@127.0.0.1:3445/postgres?sslmode=disable'
 export TEST_DATABASE_COCKROACHDB='cockroach://root@127.0.0.1:3446/defaultdb?sslmode=disable'
 ```
 
@@ -475,8 +475,8 @@ or if you would like to test one of the databases:
 
 ```shell script
 make resetdb
-export TEST_DATABASE_MYSQL='mysql://root:secret@(127.0.0.1:3444)/mysql?parseTime=true'
-export TEST_DATABASE_POSTGRESQL='postgres://postgres:secret@127.0.0.1:3445/hydra?sslmode=disable'
+export TEST_DATABASE_MYSQL='mysql://root:secret@(127.0.0.1:3444)/mysql?parseTime=true&multiStatements=true'
+export TEST_DATABASE_POSTGRESQL='postgres://postgres:secret@127.0.0.1:3445/postgres?sslmode=disable'
 export TEST_DATABASE_COCKROACHDB='cockroach://root@127.0.0.1:3446/defaultdb?sslmode=disable'
 
 # You can test against each individual database:

client/client.go

@@ -195,6 +195,10 @@ type Client struct {
 	Metadata sqlxx.JSONRawMessage `json:"metadata,omitempty" db:"metadata"`
 }
 
+func (Client) TableName() string {
+	return "hydra_client"
+}
+
 func (c *Client) GetID() string {
 	return c.ClientID
 }

client/manager_sql.go

@@ -28,21 +28,12 @@ import (
 
 	"github.com/jmoiron/sqlx"
 	"github.com/pkg/errors"
-	migrate "github.com/rubenv/sql-migrate"
-	"github.com/sirupsen/logrus"
 
 	"github.com/ory/fosite"
 	"github.com/ory/hydra/x"
-	"github.com/ory/x/dbal"
 	"github.com/ory/x/sqlcon"
 )
 
-var Migrations = map[string]*dbal.PackrMigrationSource{
-	dbal.DriverMySQL:       dbal.NewMustPackerMigrationSource(logrus.New(), AssetNames(), Asset, []string{"migrations/sql/shared", "migrations/sql/mysql"}, true),
-	dbal.DriverPostgreSQL:  dbal.NewMustPackerMigrationSource(logrus.New(), AssetNames(), Asset, []string{"migrations/sql/shared", "migrations/sql/postgres"}, true),
-	dbal.DriverCockroachDB: dbal.NewMustPackerMigrationSource(logrus.New(), AssetNames(), Asset, []string{"migrations/sql/cockroach"}, true),
-}
-
 func NewSQLManager(db *sqlx.DB, r InternalRegistry) *SQLManager {
 	return &SQLManager{
 		r:  r,
@@ -90,21 +81,6 @@ var sqlParams = []string{
 	"metadata",
 }
 
-func (m *SQLManager) PlanMigration(dbName string) ([]*migrate.PlannedMigration, error) {
-	migrate.SetTable("hydra_client_migration")
-	plan, _, err := migrate.PlanMigration(m.DB.DB, dbal.Canonicalize(m.DB.DriverName()), Migrations[dbName], migrate.Up, 0)
-	return plan, errors.WithStack(err)
-}
-
-func (m *SQLManager) CreateSchemas(dbName string) (int, error) {
-	migrate.SetTable("hydra_client_migration")
-	n, err := migrate.Exec(m.DB.DB, dbal.Canonicalize(m.DB.DriverName()), Migrations[dbName], migrate.Up)
-	if err != nil {
-		return 0, errors.Wrapf(err, "Could not migrate sql schema, applied %d Migrations", n)
-	}
-	return n, nil
-}
-
 func (m *SQLManager) GetConcreteClient(ctx context.Context, id string) (*Client, error) {
 	var d Client
 	if err := m.DB.GetContext(ctx, &d, m.DB.Rebind("SELECT * FROM hydra_client WHERE id=?"), id); err != nil {

client/manager_test.go

@@ -21,115 +21,40 @@
 package client_test
 
 import (
-	"flag"
 	"fmt"
-	"log"
-	"sync"
+	"github.com/ory/hydra/driver"
 	"testing"
 
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/jackc/pgx/v4/stdlib"
-	"github.com/stretchr/testify/require"
 
 	. "github.com/ory/hydra/client"
 	"github.com/ory/hydra/internal"
-	"github.com/ory/hydra/x"
-	"github.com/ory/x/sqlcon/dockertest"
 )
 
-var clientManagers = map[string]Manager{}
-var m sync.Mutex
-
-func TestMain(m *testing.M) {
-	flag.Parse()
-	runner := dockertest.Register()
-	runner.Exit(m.Run())
-}
-
-func connectToMySQL() {
-	db, err := dockertest.ConnectToTestMySQL()
-	if err != nil {
-		log.Fatalf("Could not connect to database: %v", err)
-	}
-
-	conf := internal.NewConfigurationWithDefaults()
-	reg := internal.NewRegistrySQL(conf, db)
-
-	m.Lock()
-	clientManagers["mysql"] = reg.ClientManager()
-	m.Unlock()
-}
-
-func connectToPG() {
-	db, err := dockertest.ConnectToTestPostgreSQL()
-	if err != nil {
-		log.Fatalf("Could not connect to database: %v", err)
-	}
-
-	conf := internal.NewConfigurationWithDefaults()
-	reg := internal.NewRegistrySQL(conf, db)
-
-	m.Lock()
-	clientManagers["postgres"] = reg.ClientManager()
-	m.Unlock()
-}
-
-func connectToCRDB() {
-	db, err := dockertest.ConnectToTestCockroachDB()
-	if err != nil {
-		log.Fatalf("Could not connect to database: %v", err)
-	}
-
-	conf := internal.NewConfigurationWithDefaults()
-	reg := internal.NewRegistrySQL(conf, db)
-
-	m.Lock()
-	clientManagers["cockroach"] = reg.ClientManager()
-	m.Unlock()
-}
-
 func TestManagers(t *testing.T) {
 	conf := internal.NewConfigurationWithDefaults()
-	reg := internal.NewRegistry(conf)
 
-	clientManagers["memory"] = reg.ClientManager()
+	registries := map[string]driver.Registry{
+		"memory": internal.NewRegistryMemory(conf),
+	}
 
 	if !testing.Short() {
-		dockertest.Parallel([]func(){
-			connectToPG,
-			connectToMySQL,
-			connectToCRDB,
-		})
+		registries["postgres"], registries["mysql"], registries["cockroach"], _ = internal.ConnectDatabases(t)
+		t.Log("connected")
 	}
 
-	t.Log("Creating schemas...")
-	for k, m := range clientManagers {
-		s, ok := m.(*SQLManager)
-		if ok {
-			x.CleanSQL(t, s.DB)
-			x, err := s.CreateSchemas(k)
-			if err != nil {
-				t.Fatal("Could not create schemas", err.Error())
-			} else {
-				t.Logf("Schemas created. Rows affected: %+v", x)
-			}
-			require.NoError(t, err)
-		}
-
+	for k, m := range registries {
 		t.Run("case=create-get-update-delete", func(t *testing.T) {
-			t.Run(fmt.Sprintf("db=%s", k), TestHelperCreateGetUpdateDeleteClient(k, m))
+			t.Run(fmt.Sprintf("db=%s", k), TestHelperCreateGetUpdateDeleteClient(k, m.ClientManager()))
 		})
 
 		t.Run("case=autogenerate-key", func(t *testing.T) {
-			t.Run(fmt.Sprintf("db=%s", k), TestHelperClientAutoGenerateKey(k, m))
+			t.Run(fmt.Sprintf("db=%s", k), TestHelperClientAutoGenerateKey(k, m.ClientManager()))
 		})
 
 		t.Run("case=auth-client", func(t *testing.T) {
-			t.Run(fmt.Sprintf("db=%s", k), TestHelperClientAuthenticate(k, m))
+			t.Run(fmt.Sprintf("db=%s", k), TestHelperClientAuthenticate(k, m.ClientManager()))
 		})
-
-		if ok {
-			x.CleanSQL(t, s.DB)
-		}
 	}
 }

client/sdk_test.go

@@ -76,7 +76,7 @@ func TestClientSDK(t *testing.T) {
 	conf := internal.NewConfigurationWithDefaults()
 	viper.Set(configuration.ViperKeySubjectTypesSupported, []string{"public"})
 	viper.Set(configuration.ViperKeyDefaultClientScope, []string{"foo", "bar"})
-	r := internal.NewRegistry(conf)
+	r := internal.NewRegistryMemory(conf)
 
 	router := x.NewRouterAdmin()
 	handler := client.NewHandler(r)