feat: integrated sqlite db for metadata

CHANGELOG.md

@@ -1,11 +1,17 @@
 ## unreleased
 
+### SQLite Metadata Store
+
+This release adds an embedded SQLite database for storing metadata required by the latest UI features like Notebooks and Annotations.
+
 ### Features
 
 1. [19811](https://github.com/influxdata/influxdb/pull/19811): Add Geo graph type to be able to store in Dashboard cells.
 1. [21218](https://github.com/influxdata/influxdb/pull/21218): Add the properties of a static legend for line graphs and band plots.
 1. [21367](https://github.com/influxdata/influxdb/pull/21367): List users via the API now supports pagination
 1. [21531](https://github.com/influxdata/influxdb/pull/21531): Remove feature flags for permanent UI features
+1. [21543](https://github.com/influxdata/influxdb/pull/21543): Added `influxd` configuration flag `--sqlite-path` for specifying a user-defined path to the SQLite database file
+1. [21543](https://github.com/influxdata/influxdb/pull/21543): Updated `influxd` configuration flag `--store` to work with string values `disk` or `memory`. Memory continues to store metadata in-memory for testing; `disk` will persist metadata to disk via bolt and SQLite
 
 ### Bug Fixes
 

cmd/influxd/launcher/cmd.go

@@ -16,6 +16,7 @@ import (
 	influxlogger "github.com/influxdata/influxdb/v2/logger"
 	"github.com/influxdata/influxdb/v2/nats"
 	"github.com/influxdata/influxdb/v2/pprof"
+	"github.com/influxdata/influxdb/v2/sqlite"
 	"github.com/influxdata/influxdb/v2/storage"
 	"github.com/influxdata/influxdb/v2/v1/coordinator"
 	"github.com/influxdata/influxdb/v2/vault"
@@ -127,6 +128,7 @@ type InfluxdOpts struct {
 
 	AssetsPath string
 	BoltPath   string
+	SqLitePath string
 	EnginePath string
 
 	StoreType   string
@@ -184,6 +186,7 @@ func NewOpts(viper *viper.Viper) *InfluxdOpts {
 		ReportingDisabled: false,
 
 		BoltPath:   filepath.Join(dir, bolt.DefaultFilename),
+		SqLitePath: filepath.Join(dir, sqlite.DefaultFilename),
 		EnginePath: filepath.Join(dir, "engine"),
 
 		HttpBindAddress:       ":8086",
@@ -197,7 +200,7 @@ func NewOpts(viper *viper.Viper) *InfluxdOpts {
 		ProfilingDisabled: false,
 		MetricsDisabled:   false,
 
-		StoreType:   BoltStore,
+		StoreType:   DiskStore,
 		SecretStore: BoltStore,
 
 		NatsPort:            nats.RandomPort,
@@ -237,6 +240,11 @@ func (o *InfluxdOpts) BindCliOpts() []cli.Opt {
 			Default: o.BoltPath,
 			Desc:    "path to boltdb database",
 		},
+		{
+			DestP: &o.SqLitePath,
+			Flag:  "sqlite-path",
+			Desc:  fmt.Sprintf("path to sqlite database. if not set, sqlite database will be stored in the bolt-path directory as %q.", sqlite.DefaultFilename),
+		},
 		{
 			DestP: &o.AssetsPath,
 			Flag:  "assets-path",
@@ -246,7 +254,7 @@ func (o *InfluxdOpts) BindCliOpts() []cli.Opt {
 			DestP:   &o.StoreType,
 			Flag:    "store",
 			Default: o.StoreType,
-			Desc:    "backing store for REST resources (bolt or memory)",
+			Desc:    "backing store for REST resources (disk or memory)",
 		},
 		{
 			DestP:   &o.Testing,

cmd/influxd/launcher/launcher.go

@@ -57,6 +57,8 @@ import (
 	"github.com/influxdata/influxdb/v2/session"
 	"github.com/influxdata/influxdb/v2/snowflake"
 	"github.com/influxdata/influxdb/v2/source"
+	"github.com/influxdata/influxdb/v2/sqlite"
+	sqliteMigrations "github.com/influxdata/influxdb/v2/sqlite/migrations"
 	"github.com/influxdata/influxdb/v2/storage"
 	storageflux "github.com/influxdata/influxdb/v2/storage/flux"
 	"github.com/influxdata/influxdb/v2/storage/readservice"
@@ -88,7 +90,9 @@ import (
 )
 
 const (
-	// BoltStore stores all REST resources in boltdb.
+	// DiskStore stores all REST resources to disk in boltdb and sqlite.
+	DiskStore = "disk"
+	// BoltStore also stores all REST resources to disk in boltdb and sqlite. Kept for backwards-compatibility.
 	BoltStore = "bolt"
 	// MemoryStore stores all REST resources in memory (useful for testing).
 	MemoryStore = "memory"
@@ -110,6 +114,7 @@ type Launcher struct {
 	boltClient *bolt.Client
 	kvStore    kv.SchemaStore
 	kvService  *kv.Service
+	sqlStore   *sqlite.SqlStore
 
 	// storage engine
 	engine Engine
@@ -185,6 +190,12 @@ func (m *Launcher) Shutdown(ctx context.Context) error {
 		errs = append(errs, err.Error())
 	}
 
+	m.log.Info("Stopping", zap.String("service", "sqlite"))
+	if err := m.sqlStore.Close(); err != nil {
+		m.log.Error("Failed closing sqlite", zap.Error(err))
+		errs = append(errs, err.Error())
+	}
+
 	m.log.Info("Stopping", zap.String("service", "query"))
 	if err := m.queryController.Shutdown(ctx); err != nil && err != context.Canceled {
 		m.log.Error("Failed closing query service", zap.Error(err))
@@ -270,28 +281,51 @@ func (m *Launcher) run(ctx context.Context, opts *InfluxdOpts) (err error) {
 	var flushers flushers
 	switch opts.StoreType {
 	case BoltStore:
-		store := bolt.NewKVStore(m.log.With(zap.String("service", "kvstore-bolt")), opts.BoltPath)
-		store.WithDB(m.boltClient.DB())
-		m.kvStore = store
+		m.log.Warn("Using --store=bolt is deprecated. Use --store=disk instead.")
+		fallthrough
+	case DiskStore:
+		kvStore := bolt.NewKVStore(m.log.With(zap.String("service", "kvstore-bolt")), opts.BoltPath)
+		kvStore.WithDB(m.boltClient.DB())
+		m.kvStore = kvStore
+
+		// If a sqlite-path is not specified, store sqlite db in the same directory as bolt with the default filename.
+		if opts.SqLitePath == "" {
+			opts.SqLitePath = filepath.Dir(opts.BoltPath) + "/" + sqlite.DefaultFilename
+		}
+		sqlStore, err := sqlite.NewSqlStore(opts.SqLitePath, m.log.With(zap.String("service", "sqlite")))
+		if err != nil {
+			m.log.Error("Failed opening sqlite store", zap.Error(err))
+			return err
+		}
+		m.sqlStore = sqlStore
+
 		if opts.Testing {
-			flushers = append(flushers, store)
+			flushers = append(flushers, kvStore, sqlStore)
 		}
 
 	case MemoryStore:
-		store := inmem.NewKVStore()
-		m.kvStore = store
+		kvStore := inmem.NewKVStore()
+		m.kvStore = kvStore
+
+		sqlStore, err := sqlite.NewSqlStore(sqlite.InmemPath, m.log.With(zap.String("service", "sqlite")))
+		if err != nil {
+			m.log.Error("Failed opening sqlite store", zap.Error(err))
+			return err
+		}
+		m.sqlStore = sqlStore
+
 		if opts.Testing {
-			flushers = append(flushers, store)
+			flushers = append(flushers, kvStore, sqlStore)
 		}
 
 	default:
-		err := fmt.Errorf("unknown store type %s; expected bolt or memory", opts.StoreType)
-		m.log.Error("Failed opening bolt", zap.Error(err))
+		err := fmt.Errorf("unknown store type %s; expected disk or memory", opts.StoreType)
+		m.log.Error("Failed opening metadata store", zap.Error(err))
 		return err
 	}
 
-	migrator, err := migration.NewMigrator(
-		m.log.With(zap.String("service", "migrations")),
+	boltMigrator, err := migration.NewMigrator(
+		m.log.With(zap.String("service", "bolt migrations")),
 		m.kvStore,
 		all.Migrations[:]...,
 	)
@@ -300,9 +334,17 @@ func (m *Launcher) run(ctx context.Context, opts *InfluxdOpts) (err error) {
 		return err
 	}
 
-	// apply migrations to metadata store
-	if err := migrator.Up(ctx); err != nil {
-		m.log.Error("Failed to apply migrations", zap.Error(err))
+	// apply migrations to the bolt metadata store
+	if err := boltMigrator.Up(ctx); err != nil {
+		m.log.Error("Failed to apply bolt migrations", zap.Error(err))
+		return err
+	}
+
+	sqliteMigrator := sqlite.NewMigrator(m.sqlStore, m.log.With(zap.String("service", "sqlite migrations")))
+
+	// apply migrations to the sqlite metadata store
+	if err := sqliteMigrator.Up(ctx, &sqliteMigrations.All{}); err != nil {
+		m.log.Error("Failed to apply sqlite migrations", zap.Error(err))
 		return err
 	}
 

cmd/influxd/launcher/launcher_helpers.go

@@ -28,6 +28,7 @@ import (
 	"github.com/influxdata/influxdb/v2/pkger"
 	"github.com/influxdata/influxdb/v2/query"
 	"github.com/influxdata/influxdb/v2/restore"
+	"github.com/influxdata/influxdb/v2/sqlite"
 	"github.com/influxdata/influxdb/v2/task/taskmodel"
 	"github.com/influxdata/influxdb/v2/tenant"
 	dto "github.com/prometheus/client_model/go"
@@ -125,6 +126,7 @@ func (tl *TestLauncher) Run(tb zaptest.TestingT, ctx context.Context, setters ..
 	}
 	opts.TestingAlwaysAllowSetup = true
 	opts.BoltPath = filepath.Join(tl.Path, bolt.DefaultFilename)
+	opts.SqLitePath = filepath.Join(tl.Path, sqlite.DefaultFilename)
 	opts.EnginePath = filepath.Join(tl.Path, "engine")
 	opts.HttpBindAddress = "127.0.0.1:0"
 	opts.LogLevel = zap.DebugLevel

go.mod

@@ -58,6 +58,7 @@ require (
 	github.com/kevinburke/go-bindata v3.11.0+incompatible
 	github.com/lib/pq v1.2.0 // indirect
 	github.com/mattn/go-isatty v0.0.12
+	github.com/mattn/go-sqlite3 v1.14.7
 	github.com/matttproud/golang_protobuf_extensions v1.0.1
 	github.com/mileusna/useragent v0.0.0-20190129205925-3e331f0949a5
 	github.com/mna/pigeon v1.0.1-0.20180808201053-bb0192cfc2ae

go.sum

@@ -412,8 +412,9 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
 github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
+github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104 h1:d8RFOZ2IiFtFWBcKEHAFYJcPTf0wY5q0exFNJZVWa1U=
 github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=

sqlite/migrations/0001_create_notebooks_table.sql

@@ -0,0 +1,13 @@
+-- The user_version should match the "000X" from the file name
+-- Ex: 0001_create_notebooks_table should have a user_verison of 1
+PRAGMA user_version=1;
+
+-- Create the initial table to store notebooks
+CREATE TABLE notebooks (
+  id TEXT NOT NULL PRIMARY KEY,
+  org_id TEXT NOT NULL,
+  name TEXT NOT NULL,
+  spec TEXT NOT NULL,
+  created_at TIMESTAMP,
+  updated_at TIMESTAMP
+);

sqlite/migrations/all.go

@@ -0,0 +1,13 @@
+//go:generate env GO111MODULE=on go run github.com/kevinburke/go-bindata/go-bindata -prefix "migrations/" -o ./migrations_gen.go -ignore README|go|conf -pkg migrations .
+
+package migrations
+
+type All struct{}
+
+func (s *All) ListNames() []string {
+	return AssetNames()
+}
+
+func (s *All) MustAssetString(n string) string {
+	return MustAssetString(n)
+}