db upgrading, more changes for credential management

docker/README.md

@@ -126,6 +126,7 @@ You can specify the following environment variables when issuing `docker run` co
 | `TLS_CONTACT_ADDRESS` | string |  | Optional email to use as contact for [LetsEncrypt](https://letsencrypt.org/) certificates, e.g. `jdoe@example.com`. |
 | `TLS_DOMAIN_NAME` | string |  | If non-empty, enables TLS (http**s**) and configures domain name of your container, e.g. `www.example.com`. In order for TLS to work you have to expose your HTTPS port to the Internet and correctly configure DNS. It WILL FAIL with `localhost` or unroutable IPs. |
 | `UID_ENCRYPTION_KEY` | string | `la6YsO+bNX/+XIkOqc5Svw==` | base64-encoded 16 random bytes used as an encryption key for user IDs. |
+| `UPGRADE_DB` | bool | `false` | Upgrade database schema if needed. |
 
 A convenient way to generate a desired number of random bytes and base64-encode them on Linux and Mac:
 ```

docker/tinode/Dockerfile

@@ -30,6 +30,12 @@ ENV TARGET_DB=$TARGET_DB
 # An option to reset database.
 ENV RESET_DB=false
 
+# An option to upgrade database.
+ENV UPGRADE_DB=false
+
+# Load sample data to database from data.json.
+ENV LOAD_SAMPLE_DATA=data.json
+
 # The MySQL DSN connection.
 ENV MYSQL_DSN='root@tcp(mysql)/tinode'
 

docker/tinode/entrypoint.sh

@@ -50,8 +50,8 @@ else
 	echo "" > static/firebase-init.js
 fi
 
-# Initialize the database if it has not been initialized yet or if data reset has been requested.
-./init-db --reset=${RESET_DB} --config=${CONFIG} --data=data.json | grep "usr;tino;" > /botdata/tino-password
+# Initialize the database if it has not been initialized yet or if data reset/upgrade has been requested.
+./init-db --reset=${RESET_DB} --upgrade=${UPGRADE_DB} --config=${CONFIG} --data=data.json | grep "usr;tino;" > /botdata/tino-password
 
 if [ -s /botdata/tino-password ] ; then
 	# Convert Tino's authentication credentials into a cookie file.

server/db/adapter.go

@@ -19,6 +19,8 @@ type Adapter interface {
 	Close() error
 	// IsOpen checks if the adapter is ready for use
 	IsOpen() bool
+	// GetDbVersion returns current database version.
+	GetDbVersion() (int, error)
 	// CheckDbVersion checks if the actual database version matches adapter version.
 	CheckDbVersion() error
 	// GetName returns the name of the adapter
@@ -27,6 +29,10 @@ type Adapter interface {
 	SetMaxResults(val int) error
 	// CreateDb creates the database optionally dropping an existing database first.
 	CreateDb(reset bool) error
+	// UpgradeDb upgrades database to the current adapter version.
+	UpgradeDb() error
+	// Version returns adapter version
+	Version() int
 
 	// User management
 
@@ -60,8 +66,9 @@ type Adapter interface {
 	CredGetAll(uid t.Uid, validatedOnly bool) ([]t.Credential, error)
 	// CredIsConfirmed returns true if the given credential method has been verified, false otherwise.
 	CredIsConfirmed(uid t.Uid, metod string) (bool, error)
-	// CredDel deletes credentials for the given method. If method is empty, deletes all user's credentials.
-	CredDel(uid t.Uid, method string) error
+	// CredDel deletes credentials for the given method/value. If method is empty, deletes all
+	// user's credentials.
+	CredDel(uid t.Uid, method, value string) error
 	// CredConfirm marks given credential as validated.
 	CredConfirm(uid t.Uid, method string) error
 	// CredFail increments count of failed validation attepmts for the given credentials.

server/db/mysql/adapter.go

@@ -32,7 +32,7 @@ const (
 	defaultDSN      = "root:@tcp(localhost:3306)/tinode?parseTime=true"
 	defaultDatabase = "tinode"
 
-	dbVersion = 106
+	adpVersion = 107
 
 	adapterName = "mysql"
 
@@ -104,8 +104,12 @@ func (a *adapter) IsOpen() bool {
 	return a.db != nil
 }
 
-// Read current database version
-func (a *adapter) getDbVersion() (int, error) {
+// GetDbVersion returns current database version.
+func (a *adapter) GetDbVersion() (int, error) {
+	if a.version > 0 {
+		return a.version, nil
+	}
+
 	var vers int
 	err := a.db.Get(&vers, "SELECT `value` FROM kvmeta WHERE `key`='version'")
 	if err != nil {
@@ -114,28 +118,32 @@ func (a *adapter) getDbVersion() (int, error) {
 		}
 		return -1, err
 	}
+
 	a.version = vers
 
-	return a.version, nil
+	return vers, nil
 }
 
 // CheckDbVersion checks whether the actual DB version matches the expected version of this adapter.
 func (a *adapter) CheckDbVersion() error {
-	if a.version <= 0 {
-		_, err := a.getDbVersion()
-		if err != nil {
-			return err
-		}
+	version, err := a.GetDbVersion()
+	if err != nil {
+		return err
 	}
 
-	if a.version != dbVersion {
-		return errors.New("Invalid database version " + strconv.Itoa(a.version) +
-			". Expected " + strconv.Itoa(dbVersion))
+	if version != adpVersion {
+		return errors.New("Invalid database version " + strconv.Itoa(version) +
+			". Expected " + strconv.Itoa(adpVersion))
 	}
 
 	return nil
 }
 
+// Version returns adapter version.
+func (adapter) Version() int {
+	return adpVersion
+}
+
 // GetName returns string that adapter uses to register itself with store.
 func (a *adapter) GetName() string {
 	return adapterName
@@ -364,6 +372,7 @@ func (a *adapter) CreateDb(reset bool) error {
 			id        INT NOT NULL AUTO_INCREMENT,
 			createdat DATETIME(3) NOT NULL,
 			updatedat DATETIME(3) NOT NULL,
+			deletedat DATETIME(3),
 			method    VARCHAR(16) NOT NULL,
 			value     VARCHAR(128) NOT NULL,
 			synthetic VARCHAR(192) NOT NULL,
@@ -417,13 +426,48 @@ func (a *adapter) CreateDb(reset bool) error {
 			`)`); err != nil {
 		return err
 	}
-	if _, err = tx.Exec("INSERT INTO kvmeta(`key`, `value`) VALUES('version', ?)", dbVersion); err != nil {
+	if _, err = tx.Exec("INSERT INTO kvmeta(`key`, `value`) VALUES('version', ?)", adpVersion); err != nil {
 		return err
 	}
 
 	return tx.Commit()
 }
 
+func (a *adapter) updateDbVersion(v int) error {
+	if _, err := a.db.Exec("UPDATE kvmeta SET version=? WHERE key='version'", v); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (a *adapter) UpgradeDb() error {
+	if _, err := a.GetDbVersion(); err != nil {
+		return err
+	}
+
+	if a.version == 106 {
+		// Perform database upgrade from version 106 to version 107.
+
+		if _, err := a.db.Exec("ALTER TABLE credentials ADD deletedat DATETIME(3) AFTER updatedat"); err != nil {
+			return err
+		}
+
+		if err := a.updateDbVersion(107); err != nil {
+			return err
+		}
+
+		if _, err := a.GetDbVersion(); err != nil {
+			return err
+		}
+	}
+
+	if a.version != adpVersion {
+		return errors.New("Failed to perform database upgrade to version " + strconv.Itoa(adpVersion) +
+			". DB is still at " + strconv.Itoa(a.version))
+	}
+	return nil
+}
+
 func addTags(tx *sqlx.Tx, table, keyName string, keyVal interface{}, tags []string, ignoreDups bool) error {
 
 	if len(tags) == 0 {
@@ -731,7 +775,7 @@ func (a *adapter) UserDelete(uid t.Uid, hard bool) error {
 		}
 
 		// Delete all credentials.
-		if err = credDel(tx, uid, ""); err != nil {
+		if err = credDel(tx, uid, "", ""); err != nil {
 			return err
 		}
 
@@ -2134,25 +2178,25 @@ func (a *adapter) DeviceDelete(uid t.Uid, deviceID string) error {
 
 // CredUpsert adds or updates a validation record.
 func (a *adapter) CredUpsert(cred *t.Credential) error {
+	now := t.TimeNow()
+
 	// If credential is validated it cannot be changed so assume it does not exist, just try to add it.
 	// If credential is not valiated, try to find it first: if it does not exist add it, otherwise
 	// update UpdatedAt and Resp.
 
 	userId := decodeUidString(cred.User)
 
 	// Deactivate all unverified records of this user and method.
-	_, err := a.db.Exec("UPDATE credentials SET done=-1 WHERE userid=? AND method=?", userId, cred.Method)
+	_, err := a.db.Exec("UPDATE credentials SET deletedat=? WHERE userid=? AND method=?", now, userId, cred.Method)
 
 	// Enforce uniqueness: if credential is confirmed, "method:value" must be unique.
 	// if credential is not yet confirmed, "userid:method:value" is unique.
 	synth := cred.Method + ":" + cred.Value
-	done := 1
 	if !cred.Done {
 		synth = cred.User + ":" + synth
-		done = 0
 
-		// Assume that the record exists and try to update it: mark as active, update timestamp and response value.
-		res, err := a.db.Exec("UPDATE credentials SET updatedat=?,resp=?,done=0 WHERE synthetic=?",
+		// Assume that the record exists and try to update it: undelete, update timestamp and response value.
+		res, err := a.db.Exec("UPDATE credentials SET updatedat=?,deletedat=NULL,resp=?,done=0 WHERE synthetic=?",
 			cred.UpdatedAt, cred.Resp, synth)
 		if err != nil {
 			return err
@@ -2167,7 +2211,7 @@ func (a *adapter) CredUpsert(cred *t.Credential) error {
 	// Add new record.
 	_, err = a.db.Exec("INSERT INTO credentials(createdat,updatedat,method,value,synthetic,userid,resp,done) "+
 		"VALUES(?,?,?,?,?,?,?,?)",
-		cred.CreatedAt, cred.UpdatedAt, cred.Method, cred.Value, synth, userId, cred.Resp, done)
+		cred.CreatedAt, cred.UpdatedAt, cred.Method, cred.Value, synth, userId, cred.Resp, cred.Done)
 	if isDupe(err) {
 		return t.ErrDuplicate
 	}
@@ -2178,7 +2222,7 @@ func (a *adapter) CredUpsert(cred *t.Credential) error {
 func (a *adapter) CredIsConfirmed(uid t.Uid, method string) (bool, error) {
 	var done int
 	// There could be more than one credential of the same method. We just need one.
-	err := a.db.Get(&done, "SELECT done FROM credentials WHERE userid=? AND method=? AND done=1",
+	err := a.db.Get(&done, "SELECT done FROM credentials WHERE userid=? AND method=? AND done=true",
 		store.DecodeUid(uid), method)
 	if err == sql.ErrNoRows {
 		// Nothing found, clear the error, otherwise it will be reported as internal error.
@@ -2189,20 +2233,46 @@ func (a *adapter) CredIsConfirmed(uid t.Uid, method string) (bool, error) {
 }
 
 // credDel deletes given validation method or all methods of the given user.
-func credDel(tx *sqlx.Tx, uid t.Uid, method string) error {
-	query := "DELETE FROM credentials WHERE userid=?"
+// 1. If user is being deleted, hard-delete all records (method == "")
+// 2. If one value is being deleted:
+// 2.1 Delete it if it's valiated or if there were no attempts at validation
+// (otherwise it could be used to circumvent the limit on validation attempts).
+// 2.2 In that case mark it as soft-deleted.
+func credDel(tx *sqlx.Tx, uid t.Uid, method, value string) error {
+	constraints := " WHERE userid=?"
 	args := []interface{}{store.DecodeUid(uid)}
+
 	if method != "" {
-		query += " AND method=?"
+		constraints += " AND method=?"
 		args = append(args, method)
+
+		if value != "" {
+			constraints += " AND value=?"
+			args = append(args, value)
+		}
 	}
-	_, err := tx.Exec(query, args...)
+
+	if method == "" {
+		_, err := tx.Exec("DELETE FROM credentials"+constraints, args...)
+		return err
+	}
+
+	// Case 2.1
+	if _, err := tx.Exec("DELETE FROM credentials"+constraints+" AND (done=true OR retries=0)", args...); err != nil {
+		return err
+	}
+
+	// Case 2.2
+	args = append([]interface{}{t.TimeNow()}, args...)
+	_, err := tx.Exec("UPDATE credentials SET deletedat=?"+constraints, args...)
+
 	return err
 }
 
-// CredDel deletes either all credentials of the given user and method
-// or all credentials of the given user if the method is blank.
-func (a *adapter) CredDel(uid t.Uid, method string) error {
+// CredDel deletes either credentials of the given user. If method is blank all
+// credentials are removed. If value is blank all credentials of the given the
+// method are removed.
+func (a *adapter) CredDel(uid t.Uid, method, value string) error {
 	tx, err := a.db.Beginx()
 	if err != nil {
 		return err
@@ -2213,7 +2283,7 @@ func (a *adapter) CredDel(uid t.Uid, method string) error {
 		}
 	}()
 
-	err = credDel(tx, uid, method)
+	err = credDel(tx, uid, method, value)
 	if err != nil {
 		return err
 	}
@@ -2224,7 +2294,7 @@ func (a *adapter) CredDel(uid t.Uid, method string) error {
 // CredConfirm marks given credential method as confirmed.
 func (a *adapter) CredConfirm(uid t.Uid, method string) error {
 	res, err := a.db.Exec(
-		"UPDATE credentials SET updatedat=?,done=1,synthetic=CONCAT(method,':',value) WHERE userid=? AND method=? AND done=0",
+		"UPDATE credentials SET updatedat=?,done=true,synthetic=CONCAT(method,':',value) WHERE userid=? AND method=? AND done=false",
 		t.TimeNow(), store.DecodeUid(uid), method)
 	if err != nil {
 		if isDupe(err) {
@@ -2240,7 +2310,7 @@ func (a *adapter) CredConfirm(uid t.Uid, method string) error {
 
 // CredFail increments failure count of the given validation method.
 func (a *adapter) CredFail(uid t.Uid, method string) error {
-	_, err := a.db.Exec("UPDATE credentials SET updatedat=?,retries=retries+1 WHERE userid=? AND method=? AND done=0",
+	_, err := a.db.Exec("UPDATE credentials SET updatedat=?,retries=retries+1 WHERE userid=? AND method=? AND done=false",
 		t.TimeNow(), store.DecodeUid(uid), method)
 	return err
 }
@@ -2249,7 +2319,8 @@ func (a *adapter) CredFail(uid t.Uid, method string) error {
 func (a *adapter) CredGetActive(uid t.Uid, method string) (*t.Credential, error) {
 	var cred t.Credential
 	err := a.db.Get(&cred, "SELECT createdat,updatedat,method,value,resp,done,retries "+
-		"FROM credentials WHERE userid=? AND method=? AND done=0", store.DecodeUid(uid), method)
+		"FROM credentials WHERE userid=? AND deletedat IS NULL AND method=? AND done=false",
+		store.DecodeUid(uid), method)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			err = nil
@@ -2263,11 +2334,9 @@ func (a *adapter) CredGetActive(uid t.Uid, method string) (*t.Credential, error)
 
 // CredGetAll returns credential records for the given user, all or validated only.
 func (a *adapter) CredGetAll(uid t.Uid, validatedOnly bool) ([]t.Credential, error) {
-	query := "SELECT createdat,updatedat,method,value,resp,done,retries FROM credentials WHERE userid=? AND done"
+	query := "SELECT createdat,updatedat,method,value,resp,done,retries FROM credentials WHERE userid=? AND deletedat IS NULL"
 	if validatedOnly {
-		query += "=1"
-	} else {
-		query += ">=0"
+		query += " AND done=true"
 	}
 
 	var credentials []t.Credential

server/db/mysql/schema.sql

@@ -165,8 +165,9 @@ CREATE TABLE dellog(
 # User credentials
 CREATE TABLE credentials(
 	id			INT NOT NULL AUTO_INCREMENT,
-	createdat 	DATETIME(3) NOT NULL,
-	updatedat 	DATETIME(3) NOT NULL,	
+	createdat	DATETIME(3) NOT NULL,
+	updatedat	DATETIME(3) NOT NULL,
+	deletedat	DATETIME(3),
 	method 		VARCHAR(16) NOT NULL,
 	value		VARCHAR(128) NOT NULL,
 	synthetic	VARCHAR(192) NOT NULL,