make it easier to spot pages with no scrapable content (github#31421)

coolaj86 · Oct 4, 2022 · 642cb07 · 642cb07
1 parent 6468a95
commit 642cb07
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 4 deletions.
diff --git a/script/search/build-records.js b/script/search/build-records.js
@@ -26,8 +26,10 @@ export default async function buildRecords(
   indexablePages,
   pageVersion,
   languageCode,
-  redirects
+  redirects,
+  config = {}
 ) {
+  const { noMarkers } = config
   console.log(`\n\nBuilding records for index '${indexName}' (${languages[languageCode].name})`)
   const records = []
   const pages = indexablePages
@@ -61,12 +63,12 @@ export default async function buildRecords(
 
   const waiter = domwaiter(permalinks, { maxConcurrent: MAX_CONCURRENT, minTime: MIN_TIME })
     .on('page', (page) => {
-      process.stdout.write(pageMarker)
+      if (!noMarkers) process.stdout.write(pageMarker)
       const newRecord = parsePageSectionsIntoRecords(page)
       const pathArticle = page.relativePath.replace('/index.md', '').replace('.md', '')
       const popularity = (hasPopularPages && popularPages[pathArticle]) || 0.0
       newRecord.popularity = popularity
-      process.stdout.write(recordMarker)
+      if (!noMarkers) process.stdout.write(recordMarker)
       records.push(newRecord)
     })
     .on('error', (err) => {

diff --git a/script/search/parse-page-sections-into-records.js b/script/search/parse-page-sections-into-records.js
@@ -32,6 +32,11 @@ export default function parsePageSectionsIntoRecords(page) {
 
   const rootSelector = '[data-search=article-body]'
   const $root = $(rootSelector)
+  if ($root.length === 0) {
+    console.warn(`${href} has no '${rootSelector}'`)
+  } else if ($root.length > 1) {
+    console.warn(`${href} has more than one '${rootSelector}' (${$root.length})`)
+  }
 
   const $sections = $('h2', $root)
     .filter('[id]')

diff --git a/script/search/sync-search-indices.js b/script/search/sync-search-indices.js
@@ -49,6 +49,7 @@ program
   // and change where it's used to be that the default is to not generate
   // any Lunr indexes.
   .option('--no-lunr-index', `Do not generate a Lunr index, just the records file (default false)`)
+  .option('--no-markers', 'Do not print a marker for each parsed document')
   .parse(process.argv)
 
 main(program.opts())
@@ -128,6 +129,10 @@ async function main(opts) {
 
   const generateLunrIndex = !!opts.lunrIndex
 
+  const config = {
+    noMarkers: !opts.markers,
+  }
+
   const options = {
     dryRun,
     language,
@@ -136,6 +141,7 @@ async function main(opts) {
     outDirectory,
     compressFiles,
     generateLunrIndex,
+    config,
   }
   await searchSync(options)
 }
diff --git a/script/search/sync.js b/script/search/sync.js
@@ -19,6 +19,7 @@ export default async function syncSearchIndexes({
   outDirectory,
   compressFiles,
   generateLunrIndex,
+  config = {},
 }) {
   const t0 = new Date()
 
@@ -72,7 +73,8 @@ export default async function syncSearchIndexes({
         indexablePages,
         pageVersion,
         languageCode,
-        redirects
+        redirects,
+        config
       )
       if (generateLunrIndex) {
         const index = new LunrIndex(indexName, records)