|
| 1 | +/** |
| 2 | + * This script is used to index the static docs HTML files generated by Next.js into Algolia. |
| 3 | + * |
| 4 | + * It's a migration from the Gatsby solution, |
| 5 | + * which relied on the `gatsby-plugin-algolia`: https://github.com/getsentry/sentry-docs/blob/3c1361bdcb23a0fcee1f3019bca7c14a5d632162/src/gatsby/utils/algolia.ts |
| 6 | + * |
| 7 | + * The record generation logic is reused as is, with *two* notable changes: |
| 8 | + * 1. We manually feed the HTML files to the record generation function |
| 9 | + * 2. We manually upload the records to Algolia |
| 10 | + * |
| 11 | + * This script is meant to be run on a GitHub Action (see `.github/workflows/algolia-index.yml`). |
| 12 | + * |
| 13 | + * If you want to run it locally, |
| 14 | + * 1. Make sure you have the required env vars set up |
| 15 | + * 2. be careful to change to `ALGOLIA_INDEX_PREFIX` to value different from `sentry-` |
| 16 | + * to avoid nuking the production index |
| 17 | + * 3. Run a production build of the app before running this script |
| 18 | + */ |
| 19 | + |
| 20 | +import fs from 'fs'; |
| 21 | +import {join} from 'path'; |
| 22 | + |
| 23 | +import {extrapolate, htmlToAlgoliaRecord} from '@sentry-internal/global-search'; |
| 24 | +import algoliasearch from 'algoliasearch'; |
| 25 | + |
| 26 | +import {getDocsFrontMatter} from '../src/mdx'; |
| 27 | +import {FrontMatter} from '../src/types'; |
| 28 | + |
| 29 | +// This is the path to the static files generated by Next.js for the app directory |
| 30 | +// The directory structure is not documented and could change in the future |
| 31 | +// The ideal way to do this is probably to run production server and fetch the HTML from there. |
| 32 | +const staticHtmlFilesPath = join(process.cwd(), '.next', 'server', 'app'); |
| 33 | + |
| 34 | +const ALGOLIA_APP_ID = process.env.ALGOLIA_APP_ID; |
| 35 | +const ALGOLIA_API_KEY = process.env.ALGOLIA_API_KEY; |
| 36 | +const ALGOLIA_INDEX_PREFIX = process.env.ALGOLIA_INDEX_PREFIX; |
| 37 | + |
| 38 | +if (!ALGOLIA_APP_ID) { |
| 39 | + throw new Error('`ALGOLIA_APP_ID` env var must be configured in repo secrets'); |
| 40 | +} |
| 41 | +if (!ALGOLIA_API_KEY) { |
| 42 | + throw new Error('`ALGOLIA_API_KEY` env var must be configured in repo secrets'); |
| 43 | +} |
| 44 | +if (!ALGOLIA_INDEX_PREFIX) { |
| 45 | + throw new Error('`ALGOLIA_INDEX_PREFIX` env var must be configured in repo secrets'); |
| 46 | +} |
| 47 | + |
| 48 | +const indexName = `${ALGOLIA_INDEX_PREFIX}docs`; |
| 49 | +const client = algoliasearch(ALGOLIA_APP_ID, ALGOLIA_API_KEY); |
| 50 | +const index = client.initIndex(indexName); |
| 51 | + |
| 52 | +indexAndUpload(); |
| 53 | +async function indexAndUpload() { |
| 54 | + // the page front matters are the source of truth for the static doc routes |
| 55 | + // as they are used directly by generateStaticParams() on [[..path]] page |
| 56 | + const pageFrontMatters = await getDocsFrontMatter(); |
| 57 | + const records = await generateAlogliaRecords(pageFrontMatters); |
| 58 | + // eslint-disable-next-line no-console |
| 59 | + console.log('🔥 Generated %d Algolia records.', records.length); |
| 60 | + // eslint-disable-next-line no-console |
| 61 | + console.log('🔥 Saving records ...'); |
| 62 | + await index |
| 63 | + .saveObjects(records, { |
| 64 | + batchSize: 10000, |
| 65 | + autoGenerateObjectIDIfNotExist: true, |
| 66 | + }) |
| 67 | + .then(result => { |
| 68 | + // eslint-disable-next-line no-console |
| 69 | + console.log('🔥 Saved %d Algolia records', result.objectIDs.length); |
| 70 | + }); |
| 71 | +} |
| 72 | + |
| 73 | +async function generateAlogliaRecords(pageFrontMatters: FrontMatter[]) { |
| 74 | + const records = await Promise.all( |
| 75 | + pageFrontMatters |
| 76 | + .filter( |
| 77 | + frontMatter => !frontMatter.draft && !frontMatter.noindex && frontMatter.title |
| 78 | + ) |
| 79 | + .map(pageFm => { |
| 80 | + // eslint-disable-next-line no-console |
| 81 | + console.log('processing:', pageFm.slug); |
| 82 | + |
| 83 | + const htmlFile = join(staticHtmlFilesPath, pageFm.slug + '.html'); |
| 84 | + const html = fs.readFileSync(htmlFile).toString(); |
| 85 | + |
| 86 | + const pageRecords = htmlToAlgoliaRecord( |
| 87 | + html, |
| 88 | + { |
| 89 | + title: pageFm.title, |
| 90 | + url: '/' + pageFm.slug + '/', |
| 91 | + pathSegments: extrapolate(pageFm.slug, '/').map(x => `/${x}/`), |
| 92 | + keywords: pageFm.keywords, |
| 93 | + }, |
| 94 | + '#main' |
| 95 | + ); |
| 96 | + |
| 97 | + return pageRecords; |
| 98 | + }) |
| 99 | + ); |
| 100 | + |
| 101 | + return records.flat(); |
| 102 | +} |
0 commit comments