store/redis: redis backend for hydra (#313)

Signed-off-by: Son Dinh <son.dinh@blacksquaremedia.com>

* oauth2: Add Redis manager
* jwk: Add Redis manager
* cmd/server: Add Redis handlers to factories
* config: Add Redis connections
* core: Update documentation; update Redis deps
* docker: Add redis container to compose
* oauth2/redis: Remove tokens signatures from set store on revoke
* cmd/host: Change Redis documentation port to database default
* docker: Comment out non-default Hydra backends on compose

client/manager_redis.go

@@ -0,0 +1,143 @@
+package client
+
+import (
+	"encoding/json"
+
+	"github.com/imdario/mergo"
+	"github.com/ory-am/fosite"
+	"github.com/ory-am/hydra/pkg"
+	"github.com/pborman/uuid"
+	"github.com/pkg/errors"
+	"gopkg.in/redis.v5"
+)
+
+type RedisManager struct {
+	DB        *redis.Client
+	Hasher    fosite.Hasher
+	KeyPrefix string
+}
+
+const redisClientTemplate = "hydra:client"
+
+func (m *RedisManager) redisClientKey() string {
+	return m.KeyPrefix + redisClientTemplate
+}
+
+func (m *RedisManager) GetConcreteClient(id string) (*Client, error) {
+	resp, err := m.DB.HGet(m.redisClientKey(), id).Bytes()
+	if err == redis.Nil {
+		return nil, errors.Wrap(pkg.ErrNotFound, "")
+	} else if err != nil {
+		return nil, errors.Wrap(err, "")
+	}
+
+	var d Client
+	if err := json.Unmarshal(resp, &d); err != nil {
+		return nil, errors.Wrap(err, "")
+	}
+
+	return &d, nil
+}
+
+func (m *RedisManager) GetClient(id string) (fosite.Client, error) {
+	return m.GetConcreteClient(id)
+}
+
+func (m *RedisManager) UpdateClient(c *Client) error {
+	o, err := m.GetClient(c.ID)
+	if err != nil {
+		return err
+	}
+
+	if c.Secret == "" {
+		c.Secret = string(o.GetHashedSecret())
+	} else {
+		h, err := m.Hasher.Hash([]byte(c.Secret))
+		if err != nil {
+			return errors.Wrap(err, "")
+		}
+		c.Secret = string(h)
+	}
+	if err := mergo.Merge(c, o); err != nil {
+		return errors.Wrap(err, "")
+	}
+
+	s, err := json.Marshal(c)
+	if err != nil {
+		return errors.Wrap(err, "")
+	}
+
+	if err := m.DB.HSet(m.redisClientKey(), c.ID, string(s)).Err(); err != nil {
+		return errors.Wrap(err, "")
+	}
+
+	return nil
+}
+
+func (m *RedisManager) Authenticate(id string, secret []byte) (*Client, error) {
+	c, err := m.GetConcreteClient(id)
+	if err != nil {
+		return nil, errors.Wrap(err, "")
+	}
+
+	if err := m.Hasher.Compare(c.GetHashedSecret(), secret); err != nil {
+		return nil, errors.Wrap(err, "")
+	}
+
+	return c, nil
+}
+
+func (m *RedisManager) CreateClient(c *Client) error {
+	if c.ID == "" {
+		c.ID = uuid.New()
+	}
+
+	hash, err := m.Hasher.Hash([]byte(c.Secret))
+	if err != nil {
+		return errors.Wrap(err, "")
+	}
+	c.Secret = string(hash)
+
+	s, err := json.Marshal(c)
+	if err != nil {
+		return errors.Wrap(err, "")
+	}
+
+	if err := m.DB.HSetNX(m.redisClientKey(), c.ID, string(s)).Err(); err != nil {
+		return errors.Wrap(err, "")
+	}
+	return nil
+}
+
+func (m *RedisManager) DeleteClient(id string) error {
+	if _, err := m.DB.HDel(m.redisClientKey(), id).Result(); err != nil {
+		return errors.Wrap(err, "")
+	}
+
+	return nil
+}
+
+func (m *RedisManager) GetClients() (map[string]Client, error) {
+	clients := make(map[string]Client)
+
+	iter := m.DB.HScan(m.redisClientKey(), 0, "", 0).Iterator()
+	for iter.Next() {
+		if !iter.Next() {
+			break
+		}
+
+		resp := iter.Val()
+
+		var d Client
+		if err := json.Unmarshal([]byte(resp), &d); err != nil {
+			return nil, errors.Wrap(err, "")
+		}
+
+		clients[d.ID] = d
+	}
+	if err := iter.Err(); err != nil {
+		return nil, err
+	}
+
+	return clients, nil
+}

client/manager_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/context"
+	"gopkg.in/redis.v5"
 )
 
 var clientManagers = map[string]Storage{}
@@ -78,6 +79,7 @@ func TestMain(m *testing.M) {
 	connectToPG()
 	connectToRethinkDB()
 	connectToMySQL()
+	connectToRedis()
 
 	os.Exit(m.Run())
 }
@@ -172,6 +174,28 @@ func connectToRethinkDB() {
 	containers = append(containers, c)
 	clientManagers["rethink"] = rethinkManager
 }
+
+func connectToRedis() {
+	var db *redis.Client
+	c, err := dockertest.ConnectToRedis(15, time.Second, func(url string) bool {
+		db = redis.NewClient(&redis.Options{
+			Addr: url,
+		})
+
+		return db.Ping().Err() == nil
+	})
+
+	if err != nil {
+		log.Fatalf("Could not connect to database: %s", err)
+	}
+
+	containers = append(containers, c)
+	clientManagers["redis"] = &RedisManager{
+		DB:     db,
+		Hasher: &fosite.BCrypt{WorkFactor: 4},
+	}
+}
+
 func TestClientAutoGenerateKey(t *testing.T) {
 	for k, m := range clientManagers {
 		t.Run(fmt.Sprintf("case=%s", k), func(t *testing.T) {

cmd/host.go

@@ -44,6 +44,9 @@ CORE CONTROLS
 	- RETHINK_TLS_CERT: A pem encoded TLS certificate passed as string. Can be used instead of RETHINK_TLS_CERT_PATH.
 		Example: RETHINK_TLS_CERT_PATH="-----BEGIN CERTIFICATE-----\nMIIDZTCCAk2gAwIBAgIEV5xOtDANBgkqhkiG9w0BAQ0FADA0MTIwMAYDVQQDDClP..."
 
+  - Redis: If DATABASE_URL is a DNS starting with redis:// Redis will be used as a storage backend.
+		Example: DATABASE_URL=redis://x:password@host:6379/0
+
 - SYSTEM_SECRET: A secret that is at least 16 characters long. If none is provided, one will be generated. They key
 	is used to encrypt sensitive data using AES-GCM (256 bit) and validate HMAC signatures.
 	Example: SYSTEM_SECRET=jf89-jgklAS9gk3rkAF90dfsk

cmd/server/handler_client_factory.go

@@ -40,6 +40,12 @@ func newClientManager(c *config.Config) client.Manager {
 		}
 		m.Watch(context.Background())
 		return m
+	case *config.RedisConnection:
+		m := &client.RedisManager{
+			DB:     con.RedisSession(),
+			Hasher: ctx.Hasher,
+		}
+		return m
 	default:
 		panic("Unknown connection type.")
 	}

cmd/server/handler_jwk_factory.go

@@ -46,6 +46,15 @@ func injectJWKManager(c *config.Config) {
 		m.Watch(context.Background())
 		ctx.KeyManager = m
 		break
+	case *config.RedisConnection:
+		m := &jwk.RedisManager{
+			DB: con.RedisSession(),
+			Cipher: &jwk.AEAD{
+				c.GetSystemSecret(),
+			},
+		}
+		ctx.KeyManager = m
+		break
 	default:
 		logrus.Fatalf("Unknown connection type.")
 	}

cmd/server/handler_oauth2_factory.go

@@ -67,6 +67,13 @@ func injectFositeStore(c *config.Config, clients client.Manager) {
 		m.Watch(context.Background())
 		store = m
 		break
+	case *config.RedisConnection:
+		m := &oauth2.FositeRedisStore{
+			DB:      con.RedisSession(),
+			Manager: clients,
+		}
+		store = m
+		break
 	default:
 		panic("Unknown connection type.")
 	}

config/backend_connections.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"net/url"
 
+	"strconv"
 	"time"
 
 	"github.com/Sirupsen/logrus"
@@ -16,6 +17,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 	r "gopkg.in/dancannon/gorethink.v2"
+	"gopkg.in/redis.v5"
 	"strings"
 )
 
@@ -152,3 +154,46 @@ func (c *RethinkDBConnection) CreateTableIfNotExists(table string) {
 		logrus.Fatalf("Could not create table: %s", err)
 	}
 }
+
+type RedisConnection struct {
+	db  *redis.Client
+	URL *url.URL
+}
+
+func (c *RedisConnection) RedisSession() *redis.Client {
+	if c.db != nil {
+		return c.db
+	}
+
+	var password string
+	var database int
+	var err error
+
+	if len(c.URL.Path) <= 1 {
+		logrus.Infof("Defaulting database to 0.")
+		database = 0
+	} else {
+		database, err = strconv.Atoi(c.URL.Path[1:])
+		if err != nil {
+			logrus.Fatalf("Database must be an integer.")
+		}
+	}
+
+	if c.URL.User != nil {
+		if p, exists := c.URL.User.Password(); exists {
+			password = p
+		} else {
+			// No username, so first value is taken as password
+			password = c.URL.User.Username()
+		}
+	}
+
+	options := &redis.Options{
+		Addr:     c.URL.Host,
+		Password: password,
+		DB:       database,
+	}
+
+	c.db = redis.NewClient(options)
+	return c.db
+}

config/config.go

@@ -151,6 +151,9 @@ func (c *Config) Context() *Context {
 		case "mysql":
 			connection = &SQLConnection{URL: u}
 			break
+		case "redis":
+			connection = &RedisConnection{URL: u}
+			break
 		default:
 			logrus.Fatalf("Unkown DSN %s in DATABASE_URL: %s", u.Scheme, c.DatabaseURL)
 		}
@@ -182,6 +185,10 @@ func (c *Config) Context() *Context {
 		}
 		manager = m
 		break
+	case *RedisConnection:
+		m := ladon.NewRedisManager(con.RedisSession(), "")
+		manager = m
+		break
 	default:
 		panic("Unknown connection type.")
 	}

docker-compose.yml

@@ -8,7 +8,10 @@ services:
       dockerfile: Dockerfile-demo
     links:
       - postgresd:postgresd
-      - mysqld:mysqld
+#     Uncomment the following line to use mysql instead.
+#      - mysqld:mysqld
+#     Uncomment the following line to use redis instead.
+#      - redisd:redisd
     volumes:
       - hydravolume:/root
     ports:
@@ -21,6 +24,8 @@ services:
       - DATABASE_URL=postgres://postgres:secret@postgresd:5432/postgres?sslmode=disable
 #     Uncomment the following line to use mysql instead.
 #      - DATABASE_URL=mysql://root:secret@tcp(mysqld:3306)/mysql?parseTime=true
+#     Uncomment the following line to use redis instead.
+#      - DATABASE_URL=redis://redisd:6379/0
       - FORCE_ROOT_CLIENT_CREDENTIALS=admin:demo-password
       - ACCESS_TOKEN_LIFESPAN=${ACCESS_TOKEN_LIFESPAN}
       - ID_TOKEN_LIFESPAN=${ID_TOKEN_LIFESPAN}
@@ -48,10 +53,15 @@ services:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=secret
 
-  mysqld:
-    image: mysql:5.7
-    environment:
-      - MYSQL_ROOT_PASSWORD=secret
+#  Uncomment the following section to use mysql instead.
+#  mysqld:
+#    image: mysql:5.7
+#    environment:
+#      - MYSQL_ROOT_PASSWORD=secret
+
+#  Uncomment the following section to use redis instead.
+#  redisd:
+#    image: redis:3.2
 
 volumes:
   hydravolume: