Skip to content

Commit

Permalink
feat: split indexes by plugin and documentation versions
Browse files Browse the repository at this point in the history
  • Loading branch information
@cmfcmf
cmfcmf committed Dec 11, 2021
1 parent 392b0fd commit 2953045
Show file tree
Hide file tree
Showing 5 changed files with 185 additions and 176 deletions.
3 changes: 2 additions & 1 deletion .vscode/settings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@
"search.exclude": {
"lib/**": true,
"example-docs/**": true
}
},
"jest.jestCommandLine": "yarn test"
}
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"prepare": "husky install",
"lint": "prettier --check **/*.{js,jsx,ts,tsx,json,css,scss,md,html}",
"format": "prettier --write **/*.{js,jsx,ts,tsx,json,css,scss,md,html}",
"test": "jest packages/docusaurus-search-local",
"test": "jest packages/docusaurus-search-local/src",
"test:e2e": "playwright test e2e-tests"
},
"lint-staged": {
Expand Down
192 changes: 97 additions & 95 deletions packages/docusaurus-search-local/src/client/theme/SearchBar/index.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,36 +25,35 @@ import { usePluginData } from "@docusaurus/useGlobalData";
import type { DSLAPluginData, MyDocument } from "../../../types";

const SEARCH_INDEX_AVAILABLE = process.env.NODE_ENV === "production";
const MAX_SEARCH_RESULTS = 8;

type MyItem = {
document: MyDocument;
score: number;
terms: string[];
};

function getItemUrl({ document }: MyItem) {
function getItemUrl({ document }: MyItem): string {
const [path, hash] = document.sectionRoute.split("#");
let url = path;
let url = path!;
if (hash) {
url += "#" + hash;
}
return url;
}

function fetchIndex(baseUrl: string) {
function fetchIndex(baseUrl: string, tag: string): Promise<IndexWithDocuments> {
if (SEARCH_INDEX_AVAILABLE) {
return fetch(`${baseUrl}search-index.json`)
return fetch(`${baseUrl}search-index-${tag}.json`)
.then((content) => content.json())
.then((json) => ({
documents: json.documents as MyDocument[],
allTags: json.allTags as string[],
index: mylunr.Index.load(json.index),
}));
} else {
// The index does not exist in development, therefore load a dummy index here.
return Promise.resolve({
documents: [],
allTags: [DEFAULT_SEARCH_TAG],
index: mylunr(function () {
this.ref("id");
this.field("title");
Expand All @@ -80,7 +79,9 @@ function useContextualSearchFilters() {

const preferredVersion = docsPreferredVersionByPluginId[pluginId];

const latestVersion = allDocsData[pluginId].versions.find((v) => v.isLast)!;
const latestVersion = allDocsData[pluginId]!.versions.find(
(v) => v.isLast
)!;

const version = activeVersion ?? preferredVersion ?? latestVersion;

Expand All @@ -98,6 +99,11 @@ function useContextualSearchFilters() {
};
}

type IndexWithDocuments = {
documents: MyDocument[];
index: lunr.Index;
};

const SearchBar = () => {
const {
siteConfig: { baseUrl },
Expand All @@ -117,23 +123,41 @@ const SearchBar = () => {
tagsRef.current = tags;
}, [tags]);

const index = useRef<
| null
| "loading"
| {
documents: MyDocument[];
allTags: string[];
index: lunr.Index;
const indexes = useRef<
Record<
string,
| {
state: "loading";
callbacks: Array<(index: IndexWithDocuments) => void>;
}
| ({ state: "ready" } & IndexWithDocuments)
>
>({});

const getIndex = async (tag: string): Promise<IndexWithDocuments> => {
const index = indexes.current[tag];
switch (index?.state) {
case "ready":
return index;
case undefined: {
const callbacks: Array<(index: IndexWithDocuments) => void> = [];
indexes.current[tag] = {
state: "loading",
callbacks,
};
const index = await fetchIndex(baseUrl, tag);
callbacks.forEach((cb) => cb(index));

return (indexes.current[tag] = {
state: "ready",
...index,
});
}
>(null);

const getIndex = async () => {
if (index.current !== null && index.current !== "loading") {
// Do not load the index (again) if its already loaded or in the process of being loaded.
return index.current;
case "loading":
return new Promise<IndexWithDocuments>((resolve) => {
index.callbacks.push(resolve);
});
}
index.current = "loading";
return (index.current = await fetchIndex(baseUrl));
};

const placeholder = translate({
Expand Down Expand Up @@ -279,81 +303,59 @@ const SearchBar = () => {
return getItemUrl(item);
},
async getItems() {
const { documents, allTags, index } = await getIndex();
const tags = tagsRef.current;
const indexes = await Promise.all(
tags.map((tag) => getIndex(tag))
);

const terms = tokenize(input);
const results = index
.query((query) => {
query.term(terms, { fields: ["title"], boost: titleBoost });
query.term(terms, {
fields: ["title"],
boost: titleBoost,
wildcard: mylunr.Query.wildcard.TRAILING,
});
query.term(terms, {
fields: ["content"],
boost: contentBoost,
});
query.term(terms, {
fields: ["content"],
boost: contentBoost,
wildcard: mylunr.Query.wildcard.TRAILING,
});

if (indexDocSidebarParentCategories) {
query.term(terms, {
fields: ["sidebarParentCategories"],
boost: parentCategoriesBoost,
});
query.term(terms, {
fields: ["sidebarParentCategories"],
boost: parentCategoriesBoost,
wildcard: mylunr.Query.wildcard.TRAILING,
});
}

// We want to search all documents with whose tag is included in `searchTags`.
// Since lunr.js does not allow OR queries, we instead prohibit all other tags.
//
// https://github.com/cmfcmf/docusaurus-search-local/issues/19
const searchTags = tagsRef.current;
allTags.forEach((tag) => {
if (!searchTags.includes(tag)) {
query.term(tag, {
fields: ["tag"],
boost: 0,
presence: mylunr.Query.presence.PROHIBITED,
// Disable stemmer for tags.
usePipeline: false,

return indexes
.flatMap(({ index, documents }) =>
index
.query((query) => {
query.term(terms, {
fields: ["title"],
boost: titleBoost,
});
}
});
})
// We need to remove results with a score of 0 that occur
// when the docs are versioned and just the version matches.
.filter((result) => result.score > 0)
.slice(0, 8)
.map((result) => ({
document: documents.find(
(document) => document.id.toString() === result.ref
)!,
score: result.score,
terms,
}));

// if (!SEARCH_INDEX_AVAILABLE) {
// results.push({
// score: 0.5,
// document: {
// id: 1,
// pageTitle: "BLOG POST TITLE",
// sectionTitle: "BLOG POST TITLE",
// sectionRoute: "/blog/d-s-l-test",
// },
// terms: ["a", "b"],
// });
// }

return results;
query.term(terms, {
fields: ["title"],
boost: titleBoost,
wildcard: mylunr.Query.wildcard.TRAILING,
});
query.term(terms, {
fields: ["content"],
boost: contentBoost,
});
query.term(terms, {
fields: ["content"],
boost: contentBoost,
wildcard: mylunr.Query.wildcard.TRAILING,
});

if (indexDocSidebarParentCategories) {
query.term(terms, {
fields: ["sidebarParentCategories"],
boost: parentCategoriesBoost,
});
query.term(terms, {
fields: ["sidebarParentCategories"],
boost: parentCategoriesBoost,
wildcard: mylunr.Query.wildcard.TRAILING,
});
}
})
.slice(0, MAX_SEARCH_RESULTS)
.map((result) => ({
document: documents.find(
(document) => document.id.toString() === result.ref
)!,
score: result.score,
terms,
}))
)
.sort((a, b) => b.score - a.score)
.slice(0, MAX_SEARCH_RESULTS);
},
},
];
Expand Down
Loading

0 comments on commit 2953045

Please sign in to comment.