Fixes orama summaries (#652)

Author: micheleriva

package.json

@@ -24,7 +24,6 @@
 		"@solidjs/router": "^0.13.1",
 		"@solidjs/start": "^1.0.0-rc.0",
 		"@vinxi/plugin-mdx": "^3.7.1",
-		"cheerio": "1.0.0-rc.12",
 		"dotenv": "^16.4.5",
 		"glob": "^10.3.10",
 		"gray-matter": "^4.0.3",
@@ -44,6 +43,7 @@
 	"devDependencies": {
 		"@kobalte/tailwindcss": "^0.9.0",
 		"@mdx-js/mdx": "^2.3.0",
+		"@orama/crawly": "^0.0.4",
 		"@tailwindcss/typography": "^0.5.10",
 		"@types/node": "^20.11.28",
 		"@typescript-eslint/eslint-plugin": "^7.2.0",

pnpm-lock.yaml

@@ -1,77 +1,25 @@
 import { readFileSync } from "node:fs";
 import { globSync } from "glob";
-import { load } from "cheerio";
+import { generalPurposeCrawler } from "@orama/crawly";
 import "dotenv/config";
 
 const ORAMA_PRIVATE_API_KEY = process.env.ORAMA_PRIVATE_API_KEY;
 const ORAMA_PRIVATE_INDEX_ID = process.env.ORAMA_PRIVATE_INDEX_ID;
 
 const baseURL = new URL("../dist", import.meta.url).pathname;
 const HTMLFiles = globSync("**/*.html", { cwd: baseURL });
-const headers = ["h1", "h2", "h3", "h4", "h5", "h6"];
 
 const pagesToIndex = HTMLFiles.flatMap((file) => {
 	const path = file.replace(/\.html$/, "");
 	const pageContent = readFileSync(
 		new URL(`../dist/${file}`, import.meta.url),
 		"utf8"
 	);
-	const $ = load(pageContent);
 
-	const results = [];
-	const title = $("h1").first().text().trim();
-	let currentSection = "";
-	let currentContent = "";
-
-	$("*").each(function () {
-		const isHeader = $(this).is(headers.join(", "));
-		if (isHeader) {
-			if (currentSection && currentContent) {
-				results.push({
-					title: title,
-					content: currentContent.trim(),
-					path: getPath(path),
-					category: getCategory(getPath(path)),
-					section: currentSection,
-				});
-			}
-			if (!$(this).is("h1")) {
-				currentSection = $(this)
-					.clone()
-					.children()
-					.remove("script")
-					.end()
-					.text()
-					.trim();
-			}
-			currentContent = "";
-		} else if ($(this).is("p")) {
-			currentContent += `${$(this).clone().children().remove("script").end().text().trim()} `;
-		}
-	});
-
-	if (currentSection && currentContent) {
-		results.push({
-			title: title,
-			content: currentContent.trim(),
-			path: getPath(path),
-			category: getCategory(getPath(path)),
-			section: currentSection,
-		});
-	}
-
-	return results;
+	const productionDocsURL = `https://docs.solidjs.com/${path}`;
+	return generalPurposeCrawler(productionDocsURL, pageContent);
 });
 
-function getPath(path) {
-	return path.endsWith("index") ? path.replace(/index$/, "") : path;
-}
-
-function getCategory(path) {
-	const category = path.split("/")[0];
-	return category === "index" ? "" : category;
-}
-
 async function emptyIndex() {
 	await fetch(
 		`https://api.oramasearch.com/api/v1/webhooks/${ORAMA_PRIVATE_INDEX_ID}/snapshot`,

scripts/sync-orama.mjs

@@ -21,10 +21,12 @@ export function SearchBox() {
 		createEffect(() => {
 			RegisterSearchBox({
 				summaryGeneration: import.meta.env.VITE_ORAMA_SECURE_PROXY,
-				searchMode: "hybrid",
 				oramaInstance: client,
 				colorScheme: selectedTheme().value,
 				backdrop: true,
+				resultsMap: {
+					description: 'content'
+				},
 				themeConfig: {
 					light: {},
 					dark: {