Implement a simple full-text search service

.devcontainer/devcontainer.json

@@ -1,5 +1,12 @@
 {
     "image":"mcr.microsoft.com/devcontainers/universal:2",
     "forwardPorts": [8099],
-    "postStartCommand": "cp .codespaces.env .env & docker-compose -f docker-compose-unitest.yml up"
+    "postStartCommand": "cp .codespaces.env .env && docker-compose -f docker-compose-unittest.yml up",
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "golang.go"
+            ]
+        }
+    }
 }

.github/workflows/go.yml

@@ -31,13 +31,16 @@ jobs:
       run: make build
 
     - name: Test (PostgreSQL data store)
-      run: make alltest
+      run: make test-core
 
     - name: Change DATA_STORE to Mongo
       run: sed -i 's/DATA_STORE=pg/DATA_STORE=mongo/g' .env
 
     - name: Change DATABASE_URL to Mongo connection string
       run: sed -i 's/DATABASE_URL=user=postgres password=postgres dbname=postgres sslmode=disable/DATABASE_URL=mongodb:\/\/localhost:27017/g' .env
 
+    - name: Remove full-text index DB
+      run: rm -rf sb.fts && rm -rf backend/sb.fts
+
     - name: Test (Mongo data store)
-      run: make alltest
+      run: make test-core

.gitignore

@@ -6,6 +6,8 @@ postgres-data/
 mongodata/
 cmd/staticbackend
 database/sqlite/test.db
+search/testdata
+**/*.fts
 
 dist/
 

Makefile

@@ -23,7 +23,7 @@ thistest:
 	go test -run $(TESTNAME) --cover ./...
 
 test-core:
-	@go test --race --cover
+	@go clean -testcache && go test --race --cover
 
 test-pg:
 	@cd database/postgresql && go test --race --cover
@@ -44,6 +44,9 @@ test-intl:
 test-extra:
 	@JWT_SECRET=okdevmode go test --race --cover ./extra
 
+test-search:
+	@cd search && rm -rf testdata && go test --race --cover
+
 stripe-dev:
 	stripe listen -p sb --forward-to http://localhost:8099/stripe
 

backend/backend.go

@@ -129,6 +129,7 @@ import (
 	"github.com/staticbackendhq/core/function"
 	"github.com/staticbackendhq/core/logger"
 	"github.com/staticbackendhq/core/model"
+	"github.com/staticbackendhq/core/search"
 	"github.com/staticbackendhq/core/storage"
 	mongodrv "go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
@@ -150,7 +151,8 @@ var (
 	// Filestore initialized Storer for raw save/delete blob file
 	Filestore storage.Storer
 	// Cache initialized Volatilizer for cache and pub/sub
-	Cache cache.Volatilizer
+	Cache  cache.Volatilizer
+	Search *search.Search
 	// Log initialized Logger for all logging
 	Log *logger.Logger
 
@@ -218,6 +220,18 @@ func Setup(cfg config.AppConfig) {
 		Filestore = storage.Local{}
 	}
 
+	ftsFilename := "sb.fts"
+	if cfg.DatabaseURL == "mem" {
+		ftsFilename = "mem.fts"
+	}
+	src, err := search.New(ftsFilename, Cache)
+	if err != nil {
+		Log.Fatal().Err(err).Msg("unable to start full-text search")
+		return
+	}
+
+	Search = src
+
 	sub := &function.Subscriber{Log: Log}
 	sub.PubSub = Cache
 	sub.GetExecEnv = func(token string) (function.ExecutionEnvironment, error) {
@@ -248,6 +262,7 @@ func Setup(cfg config.AppConfig) {
 		exe.BaseName = conf.Name
 		exe.DataStore = DB
 		exe.Volatile = Cache
+		exe.Search = Search
 
 		return exe, nil
 	}

cache/observer/observer.go

@@ -2,9 +2,10 @@ package observer
 
 import (
 	"errors"
-	"github.com/staticbackendhq/core/logger"
 	"sync"
 	"time"
+
+	"github.com/staticbackendhq/core/logger"
 )
 
 type Observer interface {

db.go

@@ -3,6 +3,7 @@ package staticbackend
 import (
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -43,7 +44,7 @@ func (database *Database) dbreq(w http.ResponseWriter, r *http.Request) {
 		}
 	} else if r.Method == http.MethodGet {
 		p := r.URL.Path
-		if !strings.HasSuffix(p, "/"){
+		if !strings.HasSuffix(p, "/") {
 			p += "/"
 		}
 
@@ -478,3 +479,38 @@ func getPagination(u *url.URL) (page int64, size int64) {
 
 	return
 }
+
+type SearchData struct {
+	Col      string `json:"col"`
+	Keywords string `json:"keywords"`
+}
+
+func (database *Database) search(w http.ResponseWriter, r *http.Request) {
+	conf, auth, err := middleware.Extract(r, true)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	var data SearchData
+	if err := parseBody(r.Body, &data); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	result, err := backend.Search.Search(conf.Name, data.Col, data.Keywords)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	fmt.Println("DEBUG", result.IDs)
+
+	docs, err := backend.DB.GetDocumentsByIDs(auth, conf.Name, data.Col, result.IDs)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	respond(w, http.StatusOK, docs)
+}

db_test.go

@@ -430,3 +430,47 @@ func TestDBCreateIndex(t *testing.T) {
 	//TODO: would be nice to validate the index were created
 	// but there's no way to get a collection's indexes for now.
 }
+
+func TestDBSearchIndexAndQuery(t *testing.T) {
+	task :=
+		Task{
+			Title:   "item indexed in search catalog",
+			Created: time.Now(),
+			Count:   1,
+		}
+
+	resp := dbReq(t, db.add, "POST", "/db/tasks", task)
+	defer resp.Body.Close()
+
+	if resp.StatusCode > 299 {
+		t.Fatal(GetResponseBody(t, resp))
+	}
+
+	var createdTask Task
+	if err := parseBody(resp.Body, &createdTask); err != nil {
+		t.Fatal(err)
+	}
+
+	backend.Search.Index(dbName, "tasks", createdTask.ID, "adding this to search index")
+	backend.Search.Index(dbName, "tasks", "invalid-id", "Montreal Expos were the best in 95")
+
+	// wait for the go routine for the pubsub to complete
+	time.Sleep(3 * time.Second)
+
+	data := SearchData{Col: "tasks", Keywords: "adding search"}
+	resp2 := dbReq(t, db.search, "POST", "/search", data)
+	defer resp2.Body.Close()
+
+	if resp2.StatusCode > 299 {
+		t.Fatal(GetResponseBody(t, resp2))
+	}
+
+	var tasks []Task
+	if err := parseBody(resp2.Body, &tasks); err != nil {
+		t.Fatal(err)
+	} else if len(tasks) != 1 {
+		t.Errorf("expected to get 1 task, got %d", len(tasks))
+	} else if tasks[0].ID != createdTask.ID {
+		t.Errorf("expected task id to be %s got %s", createdTask.ID, tasks[0].ID)
+	}
+}

function/runtime.go

@@ -13,6 +13,7 @@ import (
 	"github.com/staticbackendhq/core/database"
 	"github.com/staticbackendhq/core/logger"
 	"github.com/staticbackendhq/core/model"
+	"github.com/staticbackendhq/core/search"
 
 	"github.com/dop251/goja"
 )
@@ -22,6 +23,7 @@ type ExecutionEnvironment struct {
 	BaseName  string
 	DataStore database.Persister
 	Volatile  cache.Volatilizer
+	Search    *search.Search
 	Data      model.ExecData
 
 	CurrentRun model.ExecHistory
@@ -40,6 +42,7 @@ func (env *ExecutionEnvironment) Execute(data interface{}) error {
 	env.addHelpers(vm)
 	env.addDatabaseFunctions(vm)
 	env.addVolatileFunctions(vm)
+	env.addSearch(vm)
 
 	if _, err := vm.RunString(env.Data.Code); err != nil {
 		return err
@@ -424,6 +427,59 @@ func (env *ExecutionEnvironment) addVolatileFunctions(vm *goja.Runtime) {
 	})
 }
 
+func (env *ExecutionEnvironment) addSearch(vm *goja.Runtime) {
+	vm.Set("search", func(call goja.FunctionCall) goja.Value {
+		if len(call.Arguments) != 2 {
+			return vm.ToValue(Result{Content: "argument missmatch: you need 2 arguments for search(col, keywords)"})
+		}
+
+		var col, keywords string
+		if err := vm.ExportTo(call.Argument(0), &col); err != nil {
+			return vm.ToValue(Result{Content: "the first argument should be a string"})
+		} else if err := vm.ExportTo(call.Argument(1), &keywords); err != nil {
+			return vm.ToValue(Result{Content: "the second argument should be a string"})
+		}
+
+		results, err := env.Search.Search(env.BaseName, col, keywords)
+		if err != nil {
+			return vm.ToValue(Result{Content: fmt.Sprintf("error while executing search(): %v", err)})
+		}
+
+		docs, err := env.DataStore.GetDocumentsByIDs(env.Auth, env.BaseName, col, results.IDs)
+		if err != nil {
+			return vm.ToValue(Result{Content: fmt.Sprintf("error getting document by ids from search result: %v", err)})
+		}
+
+		for _, doc := range docs {
+			if err := env.clean(doc); err != nil {
+				return vm.ToValue(Result{Content: fmt.Sprintf("error cleaning doc: %v", err)})
+			}
+		}
+
+		return vm.ToValue(Result{OK: true, Content: docs})
+	})
+	vm.Set("indexDocument", func(call goja.FunctionCall) goja.Value {
+		if len(call.Arguments) != 3 {
+			return vm.ToValue(Result{Content: "argument missmatch: you need 3 arguments for indexDocument(col, id, text)"})
+		}
+
+		var col, id, text string
+		if err := vm.ExportTo(call.Argument(0), &col); err != nil {
+			return vm.ToValue(Result{Content: "the first argument should be a string"})
+		} else if err := vm.ExportTo(call.Argument(1), &id); err != nil {
+			return vm.ToValue(Result{Content: "the second argument should be a string"})
+		} else if err := vm.ExportTo(call.Argument(1), &text); err != nil {
+			return vm.ToValue(Result{Content: "the third argument should be a string"})
+		}
+
+		if err := env.Search.Index(env.BaseName, col, id, text); err != nil {
+			return vm.ToValue(Result{Content: fmt.Sprintf("error while trying to index the document: %v", err)})
+		}
+
+		return vm.ToValue(Result{OK: true})
+	})
+}
+
 func (env *ExecutionEnvironment) complete(err error) {
 	env.CurrentRun.Completed = time.Now()
 	env.CurrentRun.Success = err == nil

go.mod

@@ -25,6 +25,24 @@ require (
 
 require (
 	cloud.google.com/go v0.75.0 // indirect
+	github.com/RoaringBitmap/roaring v0.9.4 // indirect
+	github.com/bits-and-blooms/bitset v1.2.0 // indirect
+	github.com/blevesearch/bleve/v2 v2.3.8 // indirect
+	github.com/blevesearch/bleve_index_api v1.0.5 // indirect
+	github.com/blevesearch/geo v0.1.17 // indirect
+	github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
+	github.com/blevesearch/gtreap v0.1.1 // indirect
+	github.com/blevesearch/mmap-go v1.0.4 // indirect
+	github.com/blevesearch/scorch_segment_api/v2 v2.1.4 // indirect
+	github.com/blevesearch/segment v0.9.1 // indirect
+	github.com/blevesearch/snowballstem v0.9.0 // indirect
+	github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
+	github.com/blevesearch/vellum v1.0.9 // indirect
+	github.com/blevesearch/zapx/v11 v11.3.7 // indirect
+	github.com/blevesearch/zapx/v12 v12.3.7 // indirect
+	github.com/blevesearch/zapx/v13 v13.3.7 // indirect
+	github.com/blevesearch/zapx/v14 v14.3.7 // indirect
+	github.com/blevesearch/zapx/v15 v15.3.10 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -35,24 +53,28 @@ require (
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.1.0 // indirect
+	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
 	github.com/golang/protobuf v1.4.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/jmespath/go-jmespath v0.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.13.1 // indirect
 	github.com/magefile/mage v1.9.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c // indirect
+	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.0.2 // indirect
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
+	go.etcd.io/bbolt v1.3.5 // indirect
 	go.opentelemetry.io/otel v0.15.0 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/net v0.7.0 // indirect