Merge pull request #15 from mudler/feat/external_sources

feat: external sources management

Author: mudler

README.md

@@ -203,6 +203,26 @@ curl -X DELETE $BASE_URL/collections/myCollection/entry/delete \
   -d '{"entry":"file.txt"}'
 ```
 
+- **Add External Source**:
+
+```sh
+curl -X POST $BASE_URL/collections/myCollection/sources \
+  -H "Content-Type: application/json" \
+  -d '{"url":"https://example.com", "update_interval":30}'
+```
+
+The `update_interval` is specified in minutes. If not provided, it defaults to 60 minutes.
+
+- **Remove External Source**:
+
+```sh
+curl -X DELETE $BASE_URL/collections/myCollection/sources \
+  -H "Content-Type: application/json" \
+  -d '{"url":"https://example.com"}'
+```
+
+External sources are automatically monitored and updated in the background. The content is periodically fetched and added to the collection, making it searchable through the regular search endpoint.
+
 ---
 
 ## 📝 License

main.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
+	"github.com/mudler/localrecall/rag"
 	"github.com/sashabaranov/go-openai"
 )
 
@@ -20,6 +21,7 @@ var (
 	vectorEngine     = os.Getenv("VECTOR_ENGINE")
 	maxChunkingSize  = os.Getenv("MAX_CHUNKING_SIZE")
 	apiKeys          = os.Getenv("API_KEYS")
+	sourceManager    = rag.NewSourceManager()
 )
 
 func init() {
@@ -38,6 +40,9 @@ func init() {
 	if vectorEngine == "" {
 		vectorEngine = "chromem"
 	}
+
+	// Start the source manager
+	sourceManager.Start()
 }
 
 func startAPI(listenAddress string) {

rag/chunking.go

@@ -15,7 +15,7 @@ import (
 	"jaytaylor.com/html2text"
 )
 
-func getWebPage(url string) (string, error) {
+func GetWebPage(url string) (string, error) {
 	resp, err := http.Get(url)
 	if err != nil {
 		return "", err
@@ -32,7 +32,7 @@ func getWebPage(url string) (string, error) {
 func getWebSitemap(url string) (res []string, err error) {
 	err = sitemap.ParseFromSite(url, func(e sitemap.Entry) error {
 		xlog.Info("Sitemap page: " + e.GetLocation())
-		content, err := getWebPage(e.GetLocation())
+		content, err := GetWebPage(e.GetLocation())
 		if err == nil {
 			res = append(res, content)
 		}

rag/engine.go

@@ -1,12 +1,15 @@
 package rag
 
 import (
+	"github.com/mudler/localrecall/rag/engine"
 	"github.com/mudler/localrecall/rag/types"
 )
 
 type Engine interface {
-	Store(s string, meta map[string]string) error
+	Store(s string, metadata map[string]string) (engine.Result, error)
 	Reset() error
 	Search(s string, similarEntries int) ([]types.Result, error)
 	Count() int
+	Delete(where map[string]string, whereDocuments map[string]string, ids ...string) error
+	GetByID(id string) (types.Result, error)
 }

rag/engine/chromem.go

@@ -83,20 +83,40 @@ func (c *ChromemDB) embedding() chromem.EmbeddingFunc {
 	)
 }
 
-func (c *ChromemDB) Store(s string, metadata map[string]string) error {
+func (c *ChromemDB) Store(s string, metadata map[string]string) (Result, error) {
 	defer func() {
 		c.index++
 	}()
 	if s == "" {
-		return fmt.Errorf("empty string")
+		return Result{}, fmt.Errorf("empty string")
 	}
-	return c.collection.AddDocuments(context.Background(), []chromem.Document{
+
+	if err := c.collection.AddDocuments(context.Background(), []chromem.Document{
 		{
 			Metadata: metadata,
 			Content:  s,
 			ID:       fmt.Sprint(c.index),
 		},
-	}, runtime.NumCPU())
+	}, runtime.NumCPU()); err != nil {
+		return Result{}, err
+	}
+
+	return Result{
+		ID: fmt.Sprint(c.index),
+	}, nil
+}
+
+func (c *ChromemDB) Delete(where map[string]string, whereDocuments map[string]string, ids ...string) error {
+	return c.collection.Delete(context.Background(), where, whereDocuments, ids...)
+}
+
+func (c *ChromemDB) GetByID(id string) (types.Result, error) {
+	res, err := c.collection.GetByID(context.Background(), id)
+	if err != nil {
+		return types.Result{}, err
+	}
+
+	return types.Result{ID: res.ID, Metadata: res.Metadata, Content: res.Content}, nil
 }
 
 func (c *ChromemDB) Search(s string, similarEntries int) ([]types.Result, error) {

rag/engine/localai.go

@@ -31,19 +31,19 @@ func (db *LocalAIRAGDB) Count() int {
 	return 0
 }
 
-func (db *LocalAIRAGDB) Store(s string, metadata map[string]string) error {
+func (db *LocalAIRAGDB) Store(s string, metadata map[string]string) (Result, error) {
 	resp, err := db.openaiClient.CreateEmbeddings(context.TODO(),
 		openai.EmbeddingRequestStrings{
 			Input: []string{s},
 			Model: openai.EmbeddingModel(db.embeddingModel),
 		},
 	)
 	if err != nil {
-		return fmt.Errorf("error getting keys: %v", err)
+		return Result{}, fmt.Errorf("error getting keys: %v", err)
 	}
 
 	if len(resp.Data) == 0 {
-		return fmt.Errorf("no response from OpenAI API")
+		return Result{}, fmt.Errorf("no response from OpenAI API")
 	}
 
 	embedding := resp.Data[0].Embedding
@@ -54,10 +54,22 @@ func (db *LocalAIRAGDB) Store(s string, metadata map[string]string) error {
 	}
 	err = db.client.Set(setReq)
 	if err != nil {
-		return fmt.Errorf("error setting keys: %v", err)
+		return Result{}, fmt.Errorf("error setting keys: %v", err)
 	}
 
-	return nil
+	fmt.Println("LocalAI stores don't support IDs, so we can't delete entries once created. This is not implemented yet.")
+	return Result{
+		// TODO: LocalAI should return an ID so can be properly deleted. This is not implemented now
+		ID: "",
+	}, nil
+}
+
+func (db *LocalAIRAGDB) Delete(where map[string]string, whereDocuments map[string]string, ids ...string) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (db *LocalAIRAGDB) GetByID(id string) (types.Result, error) {
+	return types.Result{}, fmt.Errorf("not implemented")
 }
 
 func (db *LocalAIRAGDB) Search(s string, similarEntries int) ([]types.Result, error) {

rag/engine/types.go

@@ -0,0 +1,5 @@
+package engine
+
+type Result struct {
+	ID string `json:"id"`
+}