Crud APIs for dashboards (#286)

* added signoz.db to gitignore

* model and crud methods for dashboard package

* added signoz.db to dockerignore

* feat: dashboards crud WIP

* chore: moving response format to correct file

* chore: adding dependencies for sqlite3

* feat: CRUD APIs ready for dashboards

* fix: sqlite needs cgo enabled and hence need to add some flags in building go code

* feat: provision dashboards using json

* chore: mounting dashboard folder to container

.gitignore

@@ -39,3 +39,4 @@ frontend/cypress.env.json
 **/__debug_bin
 
 frontend/*.env
+pkg/query-service/signoz.db

deploy/docker/clickhouse-setup/docker-compose.yaml

@@ -27,6 +27,7 @@ services:
       - "8080:8080"
     volumes:
       - ./prometheus.yml:/root/config/prometheus.yml
+      - ../dashboards:/root/config/dashboards
 
     environment:
       - ClickHouseUrl=tcp://clickhouse:9000

pkg/query-service/.dockerignore

@@ -1,2 +1,3 @@
 .vscode
-README.md
+README.md
+signoz.db

pkg/query-service/Dockerfile

@@ -5,7 +5,7 @@ LABEL maintainer="signoz"
 
 ARG TARGETPLATFORM
 
-ENV CGO_ENABLED=0
+ENV CGO_ENABLED=1
 ENV GOPATH=/go
 
 RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \
@@ -21,7 +21,7 @@ RUN go mod download -x
 
 # Add the sources and proceed with build
 ADD . .
-RUN go build -o ./bin/query-service ./main.go
+RUN go build -a -ldflags "-linkmode external -extldflags '-static' -s -w" -o ./bin/query-service ./main.go
 RUN chmod +x ./bin/query-service
 
 # use a minimal alpine image

pkg/query-service/app/dashboards/model.go

@@ -0,0 +1,238 @@
+package dashboards
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/gosimple/slug"
+	"github.com/jmoiron/sqlx"
+	"go.signoz.io/query-service/model"
+	"go.uber.org/zap"
+)
+
+// const (
+// 	ErrorNone           ErrorType = ""
+// 	ErrorTimeout        ErrorType = "timeout"
+// 	ErrorCanceled       ErrorType = "canceled"
+// 	ErrorExec           ErrorType = "execution"
+// 	ErrorBadData        ErrorType = "bad_data"
+// 	ErrorInternal       ErrorType = "internal"
+// 	ErrorUnavailable    ErrorType = "unavailable"
+// 	ErrorNotFound       ErrorType = "not_found"
+// 	ErrorNotImplemented ErrorType = "not_implemented"
+// )
+
+// This time the global variable is unexported.
+var db *sqlx.DB
+
+// InitDB sets up setting up the connection pool global variable.
+func InitDB(dataSourceName string) error {
+	var err error
+
+	db, err = sqlx.Open("sqlite3", dataSourceName)
+	if err != nil {
+		return err
+	}
+
+	table_schema := `CREATE TABLE IF NOT EXISTS dashboards (
+		id INTEGER PRIMARY KEY AUTOINCREMENT,
+		uuid TEXT NOT NULL UNIQUE,
+		created_at datetime NOT NULL,
+		updated_at datetime NOT NULL,
+		data TEXT NOT NULL
+	);`
+
+	_, err = db.Exec(table_schema)
+	if err != nil {
+		return fmt.Errorf("Error in creating dashboard table: ", err.Error())
+	}
+
+	return nil
+}
+
+type Dashboard struct {
+	Id        int       `json:"id" db:"id"`
+	Uuid      string    `json:"uuid" db:"uuid"`
+	Slug      string    `json:"-" db:"-"`
+	CreatedAt time.Time `json:"created_at" db:"created_at"`
+	UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
+	Title     string    `json:"-" db:"-"`
+	Data      Data      `json:"data" db:"data"`
+}
+
+type Data map[string]interface{}
+
+// func (c *Data) Value() (driver.Value, error) {
+// 	if c != nil {
+// 		b, err := json.Marshal(c)
+// 		if err != nil {
+// 			return nil, err
+// 		}
+// 		return string(b), nil
+// 	}
+// 	return nil, nil
+// }
+
+func (c *Data) Scan(src interface{}) error {
+	var data []byte
+	if b, ok := src.([]byte); ok {
+		data = b
+	} else if s, ok := src.(string); ok {
+		data = []byte(s)
+	}
+	return json.Unmarshal(data, c)
+}
+
+// CreateDashboard creates a new dashboard
+func CreateDashboard(data *map[string]interface{}) (*Dashboard, *model.ApiError) {
+	dash := &Dashboard{
+		Data: *data,
+	}
+	dash.CreatedAt = time.Now()
+	dash.UpdatedAt = time.Now()
+	dash.UpdateSlug()
+	// dash.Uuid = uuid.New().String()
+	dash.Uuid = dash.Data["uuid"].(string)
+
+	map_data, err := json.Marshal(dash.Data)
+	if err != nil {
+		zap.S().Errorf("Error in marshalling data field in dashboard: ", dash, err)
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+
+	// db.Prepare("Insert into dashboards where")
+	result, err := db.Exec("INSERT INTO dashboards (uuid, created_at, updated_at, data) VALUES ($1, $2, $3, $4)", dash.Uuid, dash.CreatedAt, dash.UpdatedAt, map_data)
+
+	if err != nil {
+		zap.S().Errorf("Error in inserting dashboard data: ", dash, err)
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+	lastInsertId, err := result.LastInsertId()
+
+	if err != nil {
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+	dash.Id = int(lastInsertId)
+
+	return dash, nil
+}
+
+func GetDashboards() (*[]Dashboard, *model.ApiError) {
+
+	dashboards := []Dashboard{}
+	query := fmt.Sprintf("SELECT * FROM dashboards;")
+
+	err := db.Select(&dashboards, query)
+	if err != nil {
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+
+	return &dashboards, nil
+}
+
+func DeleteDashboard(uuid string) *model.ApiError {
+
+	query := fmt.Sprintf("DELETE FROM dashboards WHERE uuid='%s';", uuid)
+
+	result, err := db.Exec(query)
+
+	if err != nil {
+		return &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+
+	affectedRows, err := result.RowsAffected()
+	if err != nil {
+		return &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+	if affectedRows == 0 {
+		return &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no dashboard found with uuid: %s", uuid)}
+	}
+
+	return nil
+}
+
+func GetDashboard(uuid string) (*Dashboard, *model.ApiError) {
+
+	dashboard := Dashboard{}
+	query := fmt.Sprintf("SELECT * FROM dashboards WHERE uuid='%s';", uuid)
+
+	err := db.Get(&dashboard, query)
+	if err != nil {
+		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no dashboard found with uuid: %s", uuid)}
+	}
+
+	return &dashboard, nil
+}
+
+func UpdateDashboard(data *map[string]interface{}) (*Dashboard, *model.ApiError) {
+
+	uuid := (*data)["uuid"].(string)
+
+	map_data, err := json.Marshal(data)
+	if err != nil {
+		zap.S().Errorf("Error in marshalling data field in dashboard: ", data, err)
+		return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
+	}
+
+	dashboard, apiErr := GetDashboard(uuid)
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	dashboard.UpdatedAt = time.Now()
+	dashboard.Data = *data
+
+	// db.Prepare("Insert into dashboards where")
+	_, err = db.Exec("UPDATE dashboards SET updated_at=$1, data=$2 WHERE uuid=$3 ", dashboard.UpdatedAt, map_data, dashboard.Uuid)
+
+	if err != nil {
+		zap.S().Errorf("Error in inserting dashboard data: ", data, err)
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+
+	return dashboard, nil
+}
+
+// UpdateSlug updates the slug
+func (d *Dashboard) UpdateSlug() {
+	var title string
+
+	if val, ok := d.Data["title"]; ok {
+		title = val.(string)
+	}
+
+	d.Slug = SlugifyTitle(title)
+}
+
+func IsPostDataSane(data *map[string]interface{}) error {
+
+	val, ok := (*data)["uuid"]
+	if !ok || val == nil {
+		return fmt.Errorf("uuid not found in post data")
+	}
+
+	val, ok = (*data)["title"]
+	if !ok || val == nil {
+		return fmt.Errorf("title not found in post data")
+	}
+
+	return nil
+}
+
+func SlugifyTitle(title string) string {
+	s := slug.Make(strings.ToLower(title))
+	if s == "" {
+		// If the dashboard name is only characters outside of the
+		// sluggable characters, the slug creation will return an
+		// empty string which will mess up URLs. This failsafe picks
+		// that up and creates the slug as a base64 identifier instead.
+		s = base64.RawURLEncoding.EncodeToString([]byte(title))
+		if slug.MaxLength != 0 && len(s) > slug.MaxLength {
+			s = s[:slug.MaxLength]
+		}
+	}
+	return s
+}

pkg/query-service/app/dashboards/provision.go

@@ -0,0 +1,58 @@
+package dashboards
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+
+	"go.uber.org/zap"
+)
+
+func readCurrentDir(dir string) error {
+	file, err := os.Open(dir)
+	if err != nil {
+		zap.S().Errorf("failed opening directory: %s", err)
+		return err
+	}
+	defer file.Close()
+
+	list, _ := file.Readdirnames(0) // 0 to read all files and folders
+	for _, filename := range list {
+		// fmt.Println(filename)
+		zap.S().Info("Provisioning dashboard: ", filename)
+		plan, err := ioutil.ReadFile(dir + "/" + filename)
+		if err != nil {
+			zap.S().Errorf("Creating Dashboards: Error in reading json fron file: %s\t%s", filename, err)
+			continue
+		}
+		var data map[string]interface{}
+		err = json.Unmarshal(plan, &data)
+		if err != nil {
+			zap.S().Errorf("Creating Dashboards: Error in unmarshalling json from file: %s\t%s", filename, err)
+			continue
+		}
+		err = IsPostDataSane(&data)
+		if err != nil {
+			zap.S().Infof("Creating Dashboards: Error in file: %s\t%s", filename, err)
+			continue
+		}
+
+		_, apiErr := GetDashboard(data["uuid"].(string))
+		if apiErr == nil {
+			zap.S().Infof("Creating Dashboards: Error in file: %s\t%s", filename, "Dashboard already present in database")
+			continue
+		}
+
+		_, apiErr = CreateDashboard(&data)
+		if apiErr != nil {
+			zap.S().Errorf("Creating Dashboards: Error in file: %s\t%s", filename, apiErr.Err)
+			continue
+		}
+
+	}
+	return nil
+}
+
+func LoadDashboardFiles() error {
+	return readCurrentDir("./config/dashboards")
+}