Update to incrementally process records

Author: sivakumar-kailasam

.env.example

@@ -1,8 +1,5 @@
-ALGOLIA_APP_ID=
-ALGOLIA_ADMIN_KEY=
-API_DOCS_PATH=../ember-jsonapi-docs/tmp
-GUIDES_DOCS_PATH=../guides-app/dist/content
-DEBUG=false
-DRIVER=algolia
+GUIDES_DOCS_PATH="../guides-app/dist/content"
+ALGOLIA_APP_ID=""
+ALGOLIA_ADMIN_KEY=""
 AWS_ACCESS_KEY=""
 AWS_SECRET_KEY=""

README.md

@@ -5,30 +5,27 @@
 ### Setup
 
 1. `cp .env.example .env` - Copy the example environment configuration
-2. Update the Algolia .env variables and path to the root of the generated API docs
+2. Update the Algolia .env variables and path to the root of the generated API docs (only needed for populating indexes. use the `-j` flag to write to disk during development)
 3. `yarn install` - Install dependencies
 
 ### Indexing API
 
-Because of AWS rate limits, this project currently depends on having the API documentation downloaded locally. To generate the JSON files locally, follow the excellent documentation over at [ember-jsonapi-docs](https://github.com/ember-learn/ember-jsonapi-docs#running-the-app).
+Use any valid AWS tokens to setup the `AWS_ACCESS_KEY` & `AWS_SECRET_KEY` to download the json api docs.
 
-Once generated, use the following command to reindex algolia:
+Once generated, use the following command to re-index algolia:
 `yarn start -p api`
 
 ### Indexing Guides
 
 Guides json files are currently not stored somewhere, because they are built on demand using [broccoli-static-site-json](https://github.com/stonecircle/broccoli-static-site-json) in [guides-app](https://github.com/ember-learn/guides-app).
 To Generate the JSON files locally, clone [guides-app](https://github.com/ember-learn/guides-app), and run `ember build`.
 
-Once generated, use the following command to reindex algolia:
+Once generated, use the following command to re-index algolia:
 `yarn start -p guides`
 
-
 ## .env variables
 
-1. `ALGOLIA_APP_ID` - The Algolia application ID, found in "API Keys" section of the Algolia dashboard
-2. `ALGOLIA_ADMIN_KEY` - The Algolia admin key, found in "API Keys" section of the Algolia dashboard
-3. `API_DOCS_PATH` - The path to the root of the built documentation from [ember-jsonapi-docs](https://github.com/ember-learn/ember-jsonapi-docs#running-the-app)
-3. `GUIDES_DOCS_PATH` - The path to the root of the built documentation from [guides-app](https://github.com/ember-learn/guides-app)
-4. `DEBUG` - Outputs helpful debug information
-5. `DRIVER` - Controls the type of the script's output. Available options: `algolia`, `json`. `json` can be helpful for debugging the output of the script without wasting any indexing operations on `algolia`.
+1. `GUIDES_DOCS_PATH` - The path to the root of the built documentation from [guides-app](https://github.com/ember-learn/guides-app)
+2. `ALGOLIA_APP_ID` - The Algolia application ID, found in "API Keys" section of the Algolia dashboard
+3. `ALGOLIA_ADMIN_KEY` - The Algolia admin key, found in "API Keys" section of the Algolia dashboard
+4. `AWS_ACCESS_KEY` & `AWS_SECRET_KEY` - Any valid AWS token that can be used to read our public json docs

index.js

@@ -13,6 +13,7 @@ program
     '-c, --clear-index',
     'Whether indexes of the project should be cleared while processing'
   )
+  .option('-j, --json-driver', 'Use the json driver instead of algolia')
 
 program.on('--help', function() {
   console.log(`
@@ -26,10 +27,10 @@ program.parse(process.argv)
 
 switch (program.project) {
   case 'guides':
-    runGuides()
+    runGuides(program.clearIndex, program.jsonDriver)
     break
   case 'api':
-    runApi(program.clearIndex)
+    runApi(program.clearIndex, program.jsonDriver)
     break
   default:
     throw new Error('Invalid --project property')

lib/api-docs-sync.js

@@ -1,27 +1,26 @@
-import { Promise, all as waitForAllPromises } from 'bluebird'
-import S3 from 's3'
-import ora from 'ora'
-import humanSize from 'human-size'
 import http from 'http'
 import https from 'https'
+import humanSize from 'human-size'
+import ora from 'ora'
+import S3 from 's3'
 
 // To increase s3's download & upload dir perf
 http.globalAgent.maxSockets = https.globalAgent.maxSockets = 30
 
 const { AWS_ACCESS_KEY, AWS_SECRET_KEY } = process.env
 
 const client = S3.createClient({
-  s3Options: { accessKeyId: AWS_ACCESS_KEY, secretAccessKey: AWS_SECRET_KEY }
+  s3Options: { accessKeyId: AWS_ACCESS_KEY, secretAccessKey: AWS_SECRET_KEY },
 })
 
 const jsonDocsDirDownloadOptions = {
   localDir: 'tmp/json-docs',
-  s3Params: { Bucket: 'api-docs.emberjs.com', Prefix: 'json-docs' }
+  s3Params: { Bucket: 'api-docs.emberjs.com', Prefix: 'json-docs' },
 }
 
 let revDocsDirDownloadOptions = {
   localDir: 'tmp/rev-index',
-  s3Params: { Bucket: 'api-docs.emberjs.com', Prefix: 'rev-index' }
+  s3Params: { Bucket: 'api-docs.emberjs.com', Prefix: 'rev-index' },
 }
 
 const syncDir = options => {
@@ -51,8 +50,8 @@ const syncDir = options => {
 }
 
 export default function downloadExistingDocsToLocal() {
-  return waitForAllPromises([
+  return Promise.all([
     syncDir(jsonDocsDirDownloadOptions),
-    syncDir(revDocsDirDownloadOptions)
+    syncDir(revDocsDirDownloadOptions),
   ])
 }

lib/api.js

@@ -1,46 +1,38 @@
 require('dotenv').config()
 
-import { all, resolve } from 'bluebird'
+import { readJsonSync } from 'fs-extra'
+import { difference } from 'lodash-es'
 import { compare as compareSemVers } from 'semver'
-import { difference } from 'lodash'
-
-import logger from './utils/logger'
-import drivers from './drivers'
-import { readTmpFileFactory, readTmpFileAsyncFactory } from './utils/fs'
-import schemas from './schemas'
 import downloadApiDocs from './api-docs-sync'
-
-// Get 'readTmpFile' and 'readTmpFileAsync' bound by 'api'
-const PROJECT_TYPE = 'api'
-const readTmpFile = readTmpFileFactory(PROJECT_TYPE)
-const readTmpFileAsync = readTmpFileAsyncFactory(PROJECT_TYPE)
-
-const { DRIVER } = process.env
-
-const SelectedDriver = drivers[DRIVER]
+import algoliaDriver from './drivers/algolia'
+import jsonDriver from './drivers/json'
+import schemas from './schemas'
 
 const apiIndexes = ['modules', 'classes', 'methods', 'versions']
 
-export async function run(clearIndex = false) {
-  apiIndexes.map(SelectedDriver.init)
+export async function run(clearIndex = false, useJsonDriver = false) {
+  let driver = useJsonDriver ? jsonDriver : algoliaDriver
+
+  apiIndexes.map(driver.init)
 
   if (clearIndex) {
-    await all(apiIndexes.map(SelectedDriver.clear))
+    apiIndexes.map(driver.clear)
   }
 
   await downloadApiDocs()
 
-  await all([processDocs('ember'), processDocs('ember-data')])
+  await Promise.all([
+    processDocs(driver, 'ember'),
+    processDocs(driver, 'ember-data'),
+  ])
 }
 
-async function processDocs(project) {
-  let prevIndexedVersions = await SelectedDriver.getPreviouslyIndexedVersions(
-    project
-  )
+async function processDocs(driver, project) {
+  let prevIndexedVersions = await driver.getPreviouslyIndexedVersions(project)
 
   const {
-    meta: { availableVersions }
-  } = await readTmpFileAsync(`rev-index/${project}.json`)
+    meta: { availableVersions },
+  } = readJsonSync(`./tmp/rev-index/${project}.json`)
 
   let versionsToProcess = difference(availableVersions, prevIndexedVersions)
 
@@ -60,78 +52,59 @@ async function processDocs(project) {
       )
       // Run the schema against all data stored
       .map(mapDataForVersion)
-      // Write out to selected driver.
-      .map(writeToDriver)
+      .map(content => writeToDriver(driver, content))
+
+    let versions = [...prevIndexedVersions, ...versionsToProcess].sort(
+      compareSemVers
+    )
 
-    await SelectedDriver.write(
+    await driver.write(
       'versions',
-      [
-        {
-          id: project,
-          name: project,
-          versions: [...prevIndexedVersions, ...versionsToProcess].sort(
-            compareSemVers
-          )
-        }
-      ],
+      [{ id: project, name: project, versions }],
       project
     )
   } catch (err) {
     console.log('Error:: ', err)
   }
 }
 
-/**
- * Read index file for version
- *
- * @param {string}          version     - Version of library to index
- * @param {string}          libName     - Name of library currently indexing
- * @returns {Promise}       - Returns found index file json
- */
 function readIndexFileForVersion(version, libName) {
-  const emberVersionJSONPath = `rev-index/${libName}-${version}.json`
-  logger.logBlue(`OPENING:: ${emberVersionJSONPath}`)
-  return readTmpFile(emberVersionJSONPath)
+  const emberVersionJSONPath = `./tmp/rev-index/${libName}-${version}.json`
+  console.debug(`OPENING:: ${emberVersionJSONPath}`)
+  return readJsonSync(emberVersionJSONPath)
 }
 
-/**
- * Fetch public modules and classes for version
- *
- * @param {object} versionIndexObject        - The index.json file for a given version
- * @param {string} libName                  - The name of the library to index
- * @returns {object}                        - Extended version object with public modules & classes
- */
 function fetchPublicModuleClassesForVersion(versionIndexObject, libName) {
   const publicModules = versionIndexObject.data.relationships[
     'public-modules'
   ].data.map(module => {
     // Module names are uri encoded
     const id = encodeURIComponent(module.id)
-    const modulePath = `json-docs/${libName}/${
+    const modulePath = `./tmp/json-docs/${libName}/${
       versionIndexObject.data.attributes.version
     }/modules/${versionIndexObject.meta.module[id]}.json`
 
-    logger.logBlue(`OPENING:: ${modulePath}`)
-    return readTmpFile(modulePath)
+    console.debug(`OPENING:: ${modulePath}`)
+    return readJsonSync(modulePath)
   })
 
   const publicClasses = versionIndexObject.data.relationships[
     'public-classes'
   ].data.map(classObj => {
     // Class names are uri encoded
     const id = encodeURIComponent(classObj.id)
-    const classPath = `json-docs/${libName}/${
+    const classPath = `./tmp/json-docs/${libName}/${
       versionIndexObject.data.attributes.version
     }/classes/${versionIndexObject.meta.class[id]}.json`
 
-    logger.logBlue(`OPENING:: ${classPath}`)
-    return readTmpFile(classPath)
+    console.debug(`OPENING:: ${classPath}`)
+    return readJsonSync(classPath)
   })
 
   return {
     version: versionIndexObject,
     publicModules,
-    publicClasses
+    publicClasses,
   }
 }
 
@@ -151,44 +124,29 @@ function mapDataForVersion(versionObject) {
     ...versionObject,
     methods: [...methods, ...staticFunctions],
     publicModules: versionObject.publicModules.map(schemas.moduleSchema),
-    publicClasses: versionObject.publicClasses.map(schemas.classSchema)
+    publicClasses: versionObject.publicClasses.map(schemas.classSchema),
   }
 }
 
-/**
- * Writes out to the given driver
- *
- * @param versionObject         - Object version to write out
- */
-function writeToDriver(versionObject) {
+function writeToDriver(driver, versionObject) {
   const { id } = versionObject.version.data
 
   let tokens = id.split('-')
   let version = tokens.pop()
   let projectName = tokens.join('-')
 
-  logger.logGreen(
+  console.info(
     `version: ${id}, public classes: ${
       versionObject.publicClasses.length
     }, public modules: ${versionObject.publicModules.length}, methods: ${
       versionObject.methods.length
     }`
   )
 
-  return all([
-    SelectedDriver.write(
-      'modules',
-      versionObject.publicModules,
-      projectName,
-      version
-    ),
-    SelectedDriver.write(
-      'classes',
-      versionObject.publicClasses,
-      projectName,
-      version
-    ),
-    SelectedDriver.write('methods', versionObject.methods, projectName, version)
+  return Promise.all([
+    driver.write('modules', versionObject.publicModules, projectName, version),
+    driver.write('classes', versionObject.publicClasses, projectName, version),
+    driver.write('methods', versionObject.methods, projectName, version),
   ])
 }