proto-beta: Tokenizer customization

meilisearch · Aug 9, 2023 · 1441d4a · 1441d4a
1 parent d82d350
commit 1441d4a
Show file tree

Hide file tree

Showing 10 changed files with 712 additions and 49 deletions.
diff --git a/README.md b/README.md
@@ -87,7 +87,7 @@ const client = new MeiliSearch({
 Usage in an HTML (or alike) file:
 
 ```html
-<script src='https://cdn.jsdelivr.net/npm/meilisearch@latest/dist/bundles/meilisearch.umd.js'></script>
+<script src="https://cdn.jsdelivr.net/npm/meilisearch@latest/dist/bundles/meilisearch.umd.js"></script>
 <script>
   const client = new MeiliSearch({
     host: 'http://127.0.0.1:7700',
@@ -118,7 +118,7 @@ To use `meilisearch-js` with React Native, you must also install [react-native-u
 Usage in a Deno environment:
 
 ```js
-import { MeiliSearch } from "https://esm.sh/meilisearch"
+import { MeiliSearch } from 'https://esm.sh/meilisearch'
 
 const client = new MeiliSearch({
   host: 'http://127.0.0.1:7700',
@@ -134,7 +134,6 @@ const client = new MeiliSearch({
 const { MeiliSearch } = require('meilisearch')
 // Or if you are in a ES environment
 import { MeiliSearch } from 'meilisearch'
-
 ;(async () => {
   const client = new MeiliSearch({
     host: 'http://127.0.0.1:7700',
@@ -145,12 +144,16 @@ import { MeiliSearch } from 'meilisearch'
   const index = client.index('movies')
 
   const documents = [
-      { id: 1, title: 'Carol', genres: ['Romance', 'Drama'] },
-      { id: 2, title: 'Wonder Woman', genres: ['Action', 'Adventure'] },
-      { id: 3, title: 'Life of Pi', genres: ['Adventure', 'Drama'] },
-      { id: 4, title: 'Mad Max: Fury Road', genres: ['Adventure', 'Science Fiction'] },
-      { id: 5, title: 'Moana', genres: ['Fantasy', 'Action']},
-      { id: 6, title: 'Philadelphia', genres: ['Drama'] },
+    { id: 1, title: 'Carol', genres: ['Romance', 'Drama'] },
+    { id: 2, title: 'Wonder Woman', genres: ['Action', 'Adventure'] },
+    { id: 3, title: 'Life of Pi', genres: ['Adventure', 'Drama'] },
+    {
+      id: 4,
+      title: 'Mad Max: Fury Road',
+      genres: ['Adventure', 'Science Fiction'],
+    },
+    { id: 5, title: 'Moana', genres: ['Fantasy', 'Action'] },
+    { id: 6, title: 'Philadelphia', genres: ['Drama'] },
   ]
 
   // If the index 'movies' does not exist, Meilisearch creates it when you first add the documents.
@@ -194,12 +197,9 @@ Output:
 `meilisearch-js` supports all [search parameters](https://www.meilisearch.com/docs/reference/api/search#search-parameters) described in our main documentation website.
 
 ```javascript
-await index.search(
-  'wonder',
-  {
-    attributesToHighlight: ['*']
-  }
-)
+await index.search('wonder', {
+  attributesToHighlight: ['*'],
+})
 ```
 
 ```json
@@ -229,10 +229,7 @@ await index.search(
 To enable filtering, you must first add your attributes to the [`filterableAttributes` index setting](https://www.meilisearch.com/docs/reference/api/settings#filterable-attributes).
 
 ```js
-await index.updateAttributesForFaceting([
-    'id',
-    'genres'
-  ])
+await index.updateAttributesForFaceting(['id', 'genres'])
 ```
 
 You only need to perform this operation once per index.
@@ -242,12 +239,9 @@ Note that Meilisearch rebuilds your index whenever you update `filterableAttribu
 After you configured `filterableAttributes`, you can use the [`filter` search parameter](https://www.meilisearch.com/docs/reference/api/search#filter) to refine your search:
 
 ```js
-await index.search(
-  'wonder',
-  {
-    filter: ['id > 1 AND genres = Action']
-  }
-)
+await index.search('wonder', {
+  filter: ['id > 1 AND genres = Action'],
+})
 ```
 
 ```json
@@ -256,7 +250,7 @@ await index.search(
     {
       "id": 2,
       "title": "Wonder Woman",
-      "genres": ["Action","Adventure"]
+      "genres": ["Action", "Adventure"]
     }
   ],
   "offset": 0,
@@ -272,13 +266,10 @@ await index.search(
 Placeholder search makes it possible to receive hits based on your parameters without having any query (`q`). For example, in a movies database you can run an empty query to receive all results filtered by `genre`.
 
 ```javascript
-await index.search(
-  '',
-  {
-    filter: ['genres = fantasy'],
-    facets: ['genres']
-  }
-)
+await index.search('', {
+  filter: ['genres = fantasy'],
+  facets: ['genres'],
+})
 ```
 
 ```json
@@ -287,12 +278,12 @@ await index.search(
     {
       "id": 2,
       "title": "Wonder Woman",
-      "genres": ["Action","Adventure"]
+      "genres": ["Action", "Adventure"]
     },
     {
       "id": 5,
       "title": "Moana",
-      "genres": ["Fantasy","Action"]
+      "genres": ["Fantasy", "Action"]
     }
   ],
   "offset": 0,
@@ -320,9 +311,13 @@ You can abort a pending search request by providing an [AbortSignal](https://dev
 const controller = new AbortController()
 
 index
-  .search('wonder', {}, {
-    signal: controller.signal,
-  })
+  .search(
+    'wonder',
+    {},
+    {
+      signal: controller.signal,
+    }
+  )
   .then((response) => {
     /** ... */
   })
@@ -333,7 +328,6 @@ index
 controller.abort()
 ```
 
-
 ### Using Meilisearch behind a proxy <!-- omit in toc -->
 
 #### Custom request config <!-- omit in toc -->
@@ -345,11 +339,11 @@ const client: MeiliSearch = new MeiliSearch({
   host: 'http://localhost:3000/api/meilisearch/proxy',
   requestConfig: {
     headers: {
-      Authorization: AUTH_TOKEN
+      Authorization: AUTH_TOKEN,
     },
     // OR
-    credentials: 'include'
-  }
+    credentials: 'include',
+  },
 })
 ```
 
@@ -365,11 +359,11 @@ const client: MeiliSearch = new MeiliSearch({
       url,
       data: opts?.body,
       headers: opts?.headers,
-      method: (opts?.method?.toLocaleUpperCase() as Method) ?? 'GET'
+      method: (opts?.method?.toLocaleUpperCase() as Method) ?? 'GET',
     })
 
     return response.data
-  }
+  },
 })
 ```
 
@@ -532,10 +526,8 @@ client.index('myIndex').getTasks(parameters: TasksQuery): Promise<TasksResults>
 client.index('myIndex').getTask(uid: number): Promise<Task>
 ```
 
-
 #### Wait for one task
 
-
 ##### Using the client
 
 ```ts
@@ -576,7 +568,6 @@ client.getIndexes(parameters: IndexesQuery): Promise<IndexesResults<Index[]>>
 client.getRawIndexes(parameters: IndexesQuery): Promise<IndexesResults<IndexObject[]>>
 ```
 
-
 #### [Create a new index](https://www.meilisearch.com/docs/reference/api/indexes#create-an-index)
 
 ```ts
@@ -624,11 +615,13 @@ client.index('myIndex').update(data: IndexOptions): Promise<EnqueuedTask>
 #### [Delete index](https://www.meilisearch.com/docs/reference/api/indexes#delete-an-index)
 
 ##### Using the client
+
 ```ts
 client.deleteIndex(uid): Promise<void>
 ```
 
 ##### Using the index object
+
 ```ts
 client.index('myIndex').delete(): Promise<void>
 ```
@@ -897,6 +890,66 @@ client.index('myIndex').updateTypoTolerance(typoTolerance: TypoTolerance | null)
 client.index('myIndex').resetTypoTolerance(): Promise<EnqueuedTask>
 ```
 
+### Separator tokens <!-- omit in toc -->
+
+#### Get separator tokens
+
+```ts
+client.index('myIndex').getSeparatorTokens(): Promise<SeparatorTokens>
+```
+
+#### Update separator tokens
+
+```ts
+client.index('myIndex').updateSeparatorTokens(separatorTokens: SeparatorTokens | null): Promise<EnqueuedTask>
+```
+
+#### Reset separator tokens
+
+```ts
+client.index('myIndex').resetSeparatorTokens(): Promise<EnqueuedTask>
+```
+
+### Non Separator tokens <!-- omit in toc -->
+
+#### Get non separator tokens
+
+```ts
+client.index('myIndex').getNonSeparatorTokens(): Promise<NonSeparatorTokens>
+```
+
+#### Update non separator tokens
+
+```ts
+client.index('myIndex').updateNonSeparatorTokens(nonSeparatorTokens: NonSeparatorTokens | null): Promise<EnqueuedTask>
+```
+
+#### Reset non separator tokens
+
+```ts
+client.index('myIndex').resetNonSeparatorTokens(): Promise<EnqueuedTask>
+```
+
+### Dictionary <!-- omit in toc -->
+
+#### Get dictionary
+
+```ts
+client.index('myIndex').getDictionary(): Promise<Dictionary>
+```
+
+#### Update dictionary
+
+```ts
+client.index('myIndex').updateDictionary(dictionary: Dictionary | null): Promise<EnqueuedTask>
+```
+
+#### Reset dictionary
+
+```ts
+client.index('myIndex').resetDictionary(): Promise<EnqueuedTask>
+```
+
 ### Keys <!-- omit in toc -->
 
 #### [Get keys](https://www.meilisearch.com/docs/reference/api/keys#get-all-keys)

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "meilisearch",
-  "version": "0.34.1",
+  "version": "0.34.2-tokenizer-customization.0",
   "description": "The Meilisearch JS client for Node.js and the browser.",
   "keywords": [
     "meilisearch",