Move script-only dependencies to devDependencies (github#19542)

* Move lib/search/sync.js to script/search/sync.js

* Move mdast-util-from-markdown to devDeps

* Move lib/redirects/add-redirect-to-frontmatter.js to script/helpers/

* Move mkdirp to devDeps

* Move linkinator to devDeps

* Move rimraf to devDeps

* Fix script/search/sync.js require paths

* Move lib/search/build-records.js to script/search/

* Move lib/search/find-indexable-pages to script/search/

* Fix require paths for build-records

* Fix require paths for find-indexable-pages

* Move lib/search/algolia-get-remote-index-names.js to script/search/

* Movbe lib/search/algolia-search-index.js to script/search/

* Move lib/search/lunr-search-index.js to script/search/

* Move lib/search/lunr-get-index-names.js to script/search/

* Fix Lunr search index paths

* Move lib/search/validate-records.js to script/search/

* Move is-url to devDeps

* Move lib/search/algolia-client.js to script/search/

* Move lib/search/parse-page-sections-into-records.js to script/search/

* Move lib/search/rank.js to script/search/

* Fix path to cached-index-names.json file

* Normalize require for fs.promises

package.json

@@ -50,16 +50,12 @@
     "hot-shots": "^8.3.1",
     "html-entities": "^1.2.1",
     "imurmurhash": "^0.1.4",
-    "is-url": "^1.2.4",
     "js-cookie": "^2.2.1",
     "js-yaml": "^4.1.0",
-    "linkinator": "^2.13.6",
     "liquidjs": "^9.22.1",
     "lodash": "^4.17.21",
     "lunr": "^2.3.9",
     "lunr-languages": "^1.4.0",
-    "mdast-util-from-markdown": "^0.8.4",
-    "mkdirp": "^1.0.4",
     "morgan": "^1.10.0",
     "next": "^10.2.0",
     "node-fetch": "^2.6.1",
@@ -80,7 +76,6 @@
     "remark-parse": "^7.0.2",
     "remark-rehype": "^5.0.0",
     "revalidator": "^0.3.1",
-    "rimraf": "^3.0.2",
     "rss-parser": "^3.12.0",
     "search-with-your-keyboard": "1.1.0",
     "semver": "^7.3.5",
@@ -134,16 +129,20 @@
     "http-status-code": "^2.1.0",
     "husky": "^6.0.0",
     "image-size": "^1.0.0",
+    "is-url": "^1.2.4",
     "japanese-characters": "^1.1.0",
     "javascript-stringify": "^2.1.0",
     "jest": "^26.6.3",
     "jest-expect-message": "^1.0.2",
     "jest-github-actions-reporter": "^1.0.3",
     "jest-silent-reporter": "^0.5.0",
     "jest-slow-test-reporter": "^1.0.0",
+    "linkinator": "^2.13.6",
     "make-promises-safe": "^5.1.0",
+    "mdast-util-from-markdown": "^0.8.4",
     "mime": "^2.4.4",
     "mini-css-extract-plugin": "^1.6.0",
+    "mkdirp": "^1.0.4",
     "mock-express-response": "^0.3.0",
     "mockdate": "^3.0.5",
     "nock": "^13.0.11",
@@ -155,6 +154,7 @@
     "process": "^0.11.10",
     "replace": "^1.2.1",
     "resolve-url-loader": "^4.0.0",
+    "rimraf": "^3.0.2",
     "robots-parser": "^2.3.0",
     "sass": "^1.32.13",
     "sass-loader": "^11.1.1",

script/content-migrations/remove-map-topics.js

@@ -6,7 +6,7 @@ const walk = require('walk-sync')
 const stripHtmlComments = require('strip-html-comments')
 const languages = require('../../lib/languages')
 const frontmatter = require('../../lib/read-frontmatter')
-const addRedirectToFrontmatter = require('../../lib/redirects/add-redirect-to-frontmatter')
+const addRedirectToFrontmatter = require('../helpers/add-redirect-to-frontmatter')
 
 const relativeRefRegex = /\/[a-zA-Z0-9-]+/g
 const linkString = /{% [^}]*?link.*? \/(.*?) ?%}/m

script/i18n/homogenize-frontmatter.js

@@ -7,7 +7,7 @@
 //
 // [end-readme]
 
-const fs = require('fs/promises')
+const fs = require('fs').promises
 const path = require('path')
 const matter = require('gray-matter')
 const walk = require('walk-sync')

script/move-category-to-product.js

@@ -6,7 +6,7 @@ const path = require('path')
 const walk = require('walk-sync')
 const { execSync } = require('child_process')
 const matter = require('gray-matter')
-const addRedirectToFrontmatter = require('../lib/redirects/add-redirect-to-frontmatter')
+const addRedirectToFrontmatter = require('./helpers/add-redirect-to-frontmatter')
 const contentDir = path.join(__dirname, '../content')
 
 // [start-readme]

script/reconcile-filenames-with-ids.js

@@ -7,7 +7,7 @@ const slugger = new (require('github-slugger'))()
 const entities = new (require('html-entities').XmlEntities)()
 const frontmatter = require('../lib/read-frontmatter')
 const { execSync } = require('child_process')
-const addRedirectToFrontmatter = require('../lib/redirects/add-redirect-to-frontmatter')
+const addRedirectToFrontmatter = require('./helpers/add-redirect-to-frontmatter')
 
 const contentDir = path.join(process.cwd(), 'content')
 

lib/search/algolia-get-remote-index-names.js → script/search/algolia-get-remote-index-names.js

@@ -1,4 +1,4 @@
-const { namePrefix } = require('./config')
+const { namePrefix } = require('../../lib/search/config')
 const getAlgoliaClient = require('./algolia-client')
 
 module.exports = async function getRemoteIndexNames () {

lib/search/build-records.js → script/search/build-records.js

@@ -2,7 +2,7 @@ const domwaiter = require('domwaiter')
 const eventToPromise = require('event-to-promise')
 const chalk = require('chalk')
 const parsePageSectionsIntoRecords = require('./parse-page-sections-into-records')
-const languages = require('../languages')
+const languages = require('../../lib/languages')
 const pageMarker = chalk.green('|')
 const recordMarker = chalk.grey('.')
 const port = 4002

lib/search/find-indexable-pages.js → script/search/find-indexable-pages.js

@@ -1,4 +1,4 @@
-const { loadPages } = require('../pages')
+const { loadPages } = require('../../lib/pages')
 
 module.exports = async function findIndexablePages () {
   const allPages = await loadPages()

lib/search/lunr-get-index-names.js → script/search/lunr-get-index-names.js

@@ -2,5 +2,5 @@ const fs = require('fs').promises
 const path = require('path')
 
 module.exports = async function getIndexNames () {
-  return await fs.readdir(path.join(__dirname, 'indexes'))
+  return await fs.readdir(path.join(__dirname, '../../lib/search/indexes'))
 }

lib/search/lunr-search-index.js → script/search/lunr-search-index.js

@@ -9,7 +9,7 @@ const fs = require('fs').promises
 const path = require('path')
 const rank = require('./rank')
 const validateRecords = require('./validate-records')
-const { compress } = require('./compress')
+const { compress } = require('../../lib/search/compress')
 
 module.exports = class LunrIndex {
   constructor (name, records) {
@@ -76,7 +76,7 @@ module.exports = class LunrIndex {
       .then(JSON.stringify)
       .then(compress)
       .then(content => fs.writeFile(
-        path.posix.join(__dirname, 'indexes', `${this.name}-records.json.br`),
+        path.posix.join(__dirname, '../../lib/search/indexes', `${this.name}-records.json.br`),
         content
         // Do not set to 'utf8'
       ))
@@ -86,7 +86,7 @@ module.exports = class LunrIndex {
       .then(JSON.stringify)
       .then(compress)
       .then(content => fs.writeFile(
-        path.posix.join(__dirname, 'indexes', `${this.name}.json.br`),
+        path.posix.join(__dirname, '../../lib/search/indexes', `${this.name}.json.br`),
         content
         // Do not set to 'utf8'
       ))

lib/search/parse-page-sections-into-records.js → script/search/parse-page-sections-into-records.js

@@ -8,7 +8,7 @@ const ignoredHeadingSlugs = [
   'in-this-article',
   'further-reading'
 ]
-const { maxContentLength } = require('./config')
+const { maxContentLength } = require('../../lib/search/config')
 
 module.exports = function parsePageSectionsIntoRecords (href, $) {
   const title = $('h1').text().trim()

lib/search/sync.js → script/search/sync.js

@@ -3,12 +3,12 @@ const path = require('path')
 const mkdirp = require('mkdirp').sync
 const rimraf = require('rimraf').sync
 const chalk = require('chalk')
-const languages = require('../languages')
+const languages = require('../../lib/languages')
 const buildRecords = require('./build-records')
 const findIndexablePages = require('./find-indexable-pages')
 const cacheDir = path.join(process.cwd(), './.search-cache')
-const allVersions = require('../all-versions')
-const { namePrefix } = require('./config')
+const allVersions = require('../../lib/all-versions')
+const { namePrefix } = require('../../lib/search/config')
 
 // Algolia
 const getRemoteIndexNames = require('./algolia-get-remote-index-names')
@@ -96,7 +96,7 @@ module.exports = async function syncSearchIndexes (opts = {}) {
   const remoteIndexNames = process.env.AIRGAP
     ? await getLunrIndexNames()
     : await getRemoteIndexNames()
-  const cachedIndexNamesFile = path.join(__dirname, './cached-index-names.json')
+  const cachedIndexNamesFile = path.join(__dirname, '../../lib/search/cached-index-names.json')
   fs.writeFileSync(
     cachedIndexNamesFile,
     JSON.stringify(remoteIndexNames, null, 2)

lib/search/validate-records.js → script/search/validate-records.js

@@ -2,7 +2,7 @@ const assert = require('assert')
 const { isArray, isString, inRange } = require('lodash')
 const isURL = require('is-url')
 const countArrayValues = require('count-array-values')
-const { maxRecordLength } = require('./config')
+const { maxRecordLength } = require('../../lib/search/config')
 
 module.exports = function validateRecords (name, records) {
   assert(isString(name) && name.length, '`name` is required')

script/sync-search-indices.js

@@ -12,7 +12,7 @@ require('make-promises-safe')
 main()
 
 async function main () {
-  const sync = require('../lib/search/sync')
+  const sync = require('./search/sync')
   const opts = {
     dryRun: 'DRY_RUN' in process.env,
     language: process.env.LANGUAGE,

tests/unit/search/parse-page-sections-into-records.js

@@ -1,7 +1,7 @@
 const fs = require('fs')
 const path = require('path')
 const cheerio = require('cheerio')
-const parsePageSectionsIntoRecords = require('../../../lib/search/parse-page-sections-into-records')
+const parsePageSectionsIntoRecords = require('../../../script/search/parse-page-sections-into-records')
 const fixtures = {
   pageWithSections: fs.readFileSync(path.join(__dirname, 'fixtures/page-with-sections.html'), 'utf8'),
   pageWithoutSections: fs.readFileSync(path.join(__dirname, 'fixtures/page-without-sections.html'), 'utf8')

tests/unit/search/rank.js

@@ -1,4 +1,4 @@
-const rank = require('../../../lib/search/rank')
+const rank = require('../../../script/search/rank')
 
 test('search custom rankings', () => {
   const expectedRankings = [