complete CRUD example

Author: miguelgrinberg

elasticsearch/dsl/pydantic.py

@@ -29,22 +29,28 @@ class ESMeta(BaseModel):
     primary_term: int = 0
     seq_no: int = 0
     version: int = 0
+    score: float = 0
 
 
 class _BaseModel(BaseModel):
     meta: Annotated[ESMeta, dsl.mapped_field(exclude=True)] = Field(
-        default=ESMeta(), init=False
+        default=ESMeta(),
+        init=False,
     )
 
 
 class _BaseESModelMetaclass(type(BaseModel)):  # type: ignore[misc]
     @staticmethod
-    def process_annotations(metacls: Type["_BaseESModelMetaclass"], annotations: Dict[str, Any]) -> Dict[str, Any]:
+    def process_annotations(
+        metacls: Type["_BaseESModelMetaclass"], annotations: Dict[str, Any]
+    ) -> Dict[str, Any]:
         updated_annotations = {}
         for var, ann in annotations.items():
             if isinstance(ann, type(BaseModel)):
                 # an inner Pydantic model is transformed into an Object field
-                updated_annotations[var] = metacls.make_dsl_class(metacls, dsl.InnerDoc, ann)
+                updated_annotations[var] = metacls.make_dsl_class(
+                    metacls, dsl.InnerDoc, ann
+                )
             elif (
                 hasattr(ann, "__origin__")
                 and ann.__origin__ in [list, List]
@@ -59,7 +65,12 @@ def process_annotations(metacls: Type["_BaseESModelMetaclass"], annotations: Dic
         return updated_annotations
 
     @staticmethod
-    def make_dsl_class(metacls: Type["_BaseESModelMetaclass"], dsl_class: type, pydantic_model: type, pydantic_attrs: Optional[Dict[str, Any]] = None) -> type:
+    def make_dsl_class(
+        metacls: Type["_BaseESModelMetaclass"],
+        dsl_class: type,
+        pydantic_model: type,
+        pydantic_attrs: Optional[Dict[str, Any]] = None,
+    ) -> type:
         dsl_attrs = {
             attr: value
             for attr, value in dsl_class.__dict__.items()
@@ -95,7 +106,7 @@ class BaseESModel(_BaseModel, metaclass=BaseESModelMetaclass):
 
     def to_doc(self) -> dsl.Document:
         data = self.model_dump()
-        meta = {f"_{k}": v for k, v in data.pop("meta", {}).items()}
+        meta = {f"_{k}": v for k, v in data.pop("meta", {}).items() if v}
         return self._doc(**meta, **data)
 
     @classmethod
@@ -116,7 +127,7 @@ class AsyncBaseESModel(_BaseModel, metaclass=AsyncBaseESModelMetaclass):
 
     def to_doc(self) -> dsl.AsyncDocument:
         data = self.model_dump()
-        meta = {f"_{k}": v for k, v in data.pop("meta", {}).items()}
+        meta = {f"_{k}": v for k, v in data.pop("meta", {}).items() if v}
         return self._doc(**meta, **data)
 
     @classmethod

examples/quotes/backend/quotes.py

@@ -5,7 +5,7 @@
 from typing import Annotated
 
 from fastapi import FastAPI, HTTPException
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationError
 from sentence_transformers import SentenceTransformer
 
 from elasticsearch import NotFoundError
@@ -50,6 +50,7 @@ class SearchResponse(BaseModel):
     version="1.0.0",
 )
 
+
 @app.get("/api/quotes/{id}")
 async def get_quote(id: str) -> Quote:
     doc = None
@@ -64,31 +65,64 @@ async def get_quote(id: str) -> Quote:
 
 @app.post("/api/quotes", status_code=201)
 async def create_quote(req: Quote) -> Quote:
+    embed_quotes([req])
     doc = req.to_doc()
     doc.meta.id = ""
     await doc.save(refresh=True)
     return Quote.from_doc(doc)
 
 
 @app.put("/api/quotes/{id}")
-async def update_quote(id: str, req: Quote) -> Quote:
-    doc = req.to_doc()
-    doc.meta.id = id
+async def update_quote(id: str, quote: Quote) -> Quote:
+    doc = None
+    try:
+        doc = await Quote._doc.get(id)
+    except NotFoundError:
+        pass
+    if not doc:
+        raise HTTPException(status_code=404, detail="Item not found")
+    if quote.quote:
+        embed_quotes([quote])
+        doc.quote = quote.quote
+        doc.embedding = quote.embedding
+    if quote.author:
+        doc.author = quote.author
+    if quote.tags:
+        doc.tags = quote.tags
     await doc.save(refresh=True)
     return Quote.from_doc(doc)
 
 
 @app.delete("/api/quotes/{id}", status_code=204)
-async def delete_quote(id: str, req: Quote) -> None:
-    doc = await Quote._doc.get(id)
+async def delete_quote(id: str) -> None:
+    doc = None
+    try:
+        doc = await Quote._doc.get(id)
+    except NotFoundError:
+        pass
     if not doc:
         raise HTTPException(status_code=404, detail="Item not found")
     await doc.delete(refresh=True)
 
 
 @app.post('/api/search')
 async def search_quotes(req: SearchRequest) -> SearchResponse:
-    quotes, tags, total = await search_quotes(req.query, req.filters, use_knn=req.knn, start=req.start)
+    s = Quote._doc.search()
+    if req.query == '':
+        s = s.query(dsl.query.MatchAll())
+    elif req.knn:
+        s = s.query(dsl.query.Knn(field=Quote._doc.embedding, query_vector=model.encode(req.query).tolist()))
+    else:
+        s = s.query(dsl.query.Match(quote=req.query))
+    for tag in req.filters:
+        s = s.filter(dsl.query.Terms(tags=[tag]))
+    s.aggs.bucket('tags', dsl.aggs.Terms(field=Quote._doc.tags, size=100))
+
+    r = await s[req.start:req.start + 25].execute()
+    tags = [(tag.key, tag.doc_count) for tag in r.aggs.tags.buckets]
+    quotes = [Quote.from_doc(hit) for hit in r.hits]
+    total = r['hits'].total.value
+    
     return SearchResponse(
         quotes=quotes,
         tags=[Tag(tag=t[0], count=t[1]) for t in tags],
@@ -97,6 +131,12 @@ async def search_quotes(req: SearchRequest) -> SearchResponse:
     )
 
 
+def embed_quotes(quotes):
+    embeddings = model.encode([q.quote for q in quotes])
+    for q, e in zip(quotes, embeddings):
+        q.embedding = e.tolist()
+
+
 async def ingest_quotes():
     if await Quote._doc._index.exists():
         await Quote._doc._index.delete()
@@ -106,11 +146,6 @@ def ingest_progress(count, start):
         elapsed = time() - start
         print(f'\rIngested {count} quotes. ({count / elapsed:.0f}/sec)', end='')
 
-    def embed_quotes(quotes):
-        embeddings = model.encode([q.quote for q in quotes])
-        for q, e in zip(quotes, embeddings):
-            q.embedding = e.tolist()
-
     async def get_next_quote():
         quotes: list[Quote] = []
         with open('quotes.csv') as f:
@@ -137,21 +172,5 @@ async def get_next_quote():
     await Quote._doc.bulk(get_next_quote())
 
 
-async def search_quotes(q, tags, use_knn=True, start=0, size=25):
-    s = Quote._doc.search()
-    if q == '':
-        s = s.query(dsl.query.MatchAll())
-    elif use_knn:
-        s = s.query(dsl.query.Knn(field=Quote._doc.embedding, query_vector=model.encode(q).tolist()))
-    else:
-        s = s.query(dsl.query.Match(quote=q))
-    for tag in tags:
-        s = s.filter(dsl.query.Terms(tags=[tag]))
-    s.aggs.bucket('tags', dsl.aggs.Terms(field=Quote._doc.tags, size=100))
-    r = await s[start:start + size].execute()
-    tags = [(tag.key, tag.doc_count) for tag in r.aggs.tags.buckets]
-    return [Quote.from_doc(hit) for hit in r.hits], tags, r['hits'].total.value
-
-
 if __name__ == "__main__":
     asyncio.run(ingest_quotes())

examples/quotes/package-lock.json

@@ -17,7 +17,8 @@
     "bootstrap": "^5.3.8",
     "react": "^19.1.1",
     "react-bootstrap": "^2.10.10",
-    "react-dom": "^19.1.1"
+    "react-dom": "^19.1.1",
+    "react-router": "^7.9.2"
   },
   "devDependencies": {
     "@eslint/js": "^9.36.0",

examples/quotes/package.json

@@ -1,5 +1,6 @@
 import React, { useRef, useState, useEffect } from 'react';
 import Container from 'react-bootstrap/Container';
+import { NavLink } from 'react-router';
 import Row from 'react-bootstrap/Row';
 import Col from 'react-bootstrap/Col';
 import Form from 'react-bootstrap/Form';
@@ -9,23 +10,7 @@ import Stack from 'react-bootstrap/Stack';
 import CloseButton from 'react-bootstrap/CloseButton';
 import ToggleButton from 'react-bootstrap/ToggleButton';
 import Pagination from 'react-bootstrap/Pagination';
-
-interface Meta {
-  id: string;
-  score: number;
-}
-
-interface Quote {
-  meta: Meta;
-  quote: string;
-  author: string;
-  tags: string[];
-};
-
-interface Tag {
-  tag: string;
-  count: number;
-};
+import type { Quote, Tag } from './models';
 
 export default function App() {
   const inputRef = useRef<HTMLInputElement>(null);
@@ -96,9 +81,10 @@ export default function App() {
           <Stack direction="horizontal" gap={2}>
             <Form.Control type="text" placeholder="Search for... ?" ref={inputRef} autoFocus={true} />
             <CloseButton onClick={onResetQuery} disabled={query === ''} />
-            <ToggleButton id="knn" type="checkbox" variant="outline-primary" checked={knn} value="1" title="Use hybrid search" onChange={e => setKnn(e.currentTarget.checked)}>
+            <ToggleButton id="knn" type="checkbox" variant="outline-secondary" checked={knn} value="1" title={knn ? "Using hybrid search" : "Using full-text search"} onChange={e => setKnn(e.currentTarget.checked)}>
               <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" viewBox="0 0 256 256"><path d="M192.5,171.47A88.34,88.34,0,0,0,224,101.93c-1-45.71-37.61-83.4-83.24-85.8A88,88,0,0,0,48,102L25.55,145.18c-.09.18-.18.36-.26.54a16,16,0,0,0,7.55,20.62l.25.11L56,176.94V208a16,16,0,0,0,16,16h48a8,8,0,0,0,0-16H72V171.81a8,8,0,0,0-4.67-7.28L40,152l23.07-44.34A7.9,7.9,0,0,0,64,104a72,72,0,0,1,56-70.21V49.38a24,24,0,1,0,16,0V32c1.3,0,2.6,0,3.9.1A72.26,72.26,0,0,1,203.84,80H184a8,8,0,0,0-6.15,2.88L152.34,113.5a24.06,24.06,0,1,0,12.28,10.25L187.75,96h19.79q.36,3.12.44,6.3a72.26,72.26,0,0,1-28.78,59.3,8,8,0,0,0-3.14,7.39l8,64a8,8,0,0,0,7.93,7,8.39,8.39,0,0,0,1-.06,8,8,0,0,0,6.95-8.93ZM128,80a8,8,0,1,1,8-8A8,8,0,0,1,128,80Zm16,64a8,8,0,1,1,8-8A8,8,0,0,1,144,144Z"></path></svg>
             </ToggleButton>
+            <NavLink to="/quotes/new" className="btn btn-primary">New&nbsp;Quote</NavLink>
           </Stack>
         </Form>
       </Row>
@@ -159,6 +145,8 @@ export default function App() {
                         <span className="ResultQuote">{quote}</span> — <span className="ResultAuthor">{author}</span>
                         <br />
                         <span className="ResultScore">[Score: {meta.score}]</span> <span className="ResultTags">{tags.map(tag => `#${tag}`).join(', ')}</span>
+                        <br />
+                        <small><NavLink to={`/quotes/${meta.id}`}>Edit</NavLink></small>
                       </p>
                     </div>
                   ))}