Incremental docs publish, travis builds n more enhancements

.editorconfig

@@ -0,0 +1,21 @@
+# EditorConfig helps developers define and maintain consistent
+# coding styles between different editors and IDEs
+# editorconfig.org
+
+root = true
+
+
+[*]
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+
+[*.js]
+indent_style = space
+indent_size = 2
+
+[*.{diff,md}]
+trim_trailing_whitespace = false

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 tmp
+.vscode

.travis.yml

@@ -0,0 +1,18 @@
+language: node_js
+node_js: '6'
+env:
+  - COUCH_URL="https://sivakumar-kailasam.cloudant.com/ember-docs-api"
+
+cache:
+  directories:
+    - $HOME/.yarn-cache
+    - node_modules
+    - tmp
+
+script:
+  - yarn run test
+  - yarn start
+
+branches:
+  only:
+    - master

index.js

@@ -2,16 +2,22 @@
 
 let RSVP = require('rsvp')
 let _ = require('lodash')
-var fetch = require('./lib/fetch')
-var readDocs = require('./lib/read-docs')
-var addSinceTags = require('./lib/add-since-tags')
-var addInheritedItems = require('./lib/add-inherited-items')
-var putClassesInCouch = require('./lib/classes-in-couch')
-var createVersionIndex = require('./lib/create-version-index')
-let normalizeEmberDependencies = require('./lib/normalize-ember-dependencies')
-let normalizeIDs = require('./lib/normalize-ids')
 let rm = require('rimraf')
 let PouchDB = require('pouchdb')
+let fs = require('fs')
+let glob = require('glob')
+
+let fetch = require('./lib/fetch')
+let readDocs = require('./lib/read-docs')
+let addSinceTags = require('./lib/add-since-tags')
+let addInheritedItems = require('./lib/add-inherited-items')
+let putClassesInCouch = require('./lib/classes-in-couch')
+let createVersionIndex = require('./lib/create-version-index')
+let normalizeEmberDependencies = require('./lib/normalize-ember-dependencies')
+let normalizeIDs = require('./lib/normalize-ids')
+let markup = require('./lib/markup')
+let batchUpdate = require('./lib/batch-update')
+
 require('marked')
 
 let db = new PouchDB(process.env.COUCH_URL, {
@@ -20,19 +26,14 @@ let db = new PouchDB(process.env.COUCH_URL, {
     password: process.env.COUCH_PASSWORD
   }
 })
-let fs = require('fs')
 
 if (fs.existsSync('tmp/docs')) {
   rm.sync('tmp/docs')
 }
 
-function fetchProject (projectName) {
-  console.log('downloading docs for ' + projectName)
-  let promise = fetch()
-    .then((stuff) => {
-      console.log('reading docs for ' + projectName)
-      return readDocs(projectName)
-    })
+function transformProjectFiles (projectName) {
+  console.log('reading docs for ' + projectName)
+  let promise = RSVP.resolve(readDocs(projectName))
     .then((stuff) => {
       console.log('adding since tags for ' + projectName)
       return addSinceTags(stuff)
@@ -47,33 +48,33 @@ function fetchProject (projectName) {
       return createVersionIndex(db, projectName, doc).then(() => doc)
     }).then(doc => {
       console.log('converting markdown to html for ' + projectName)
-
-      return require('./lib/markup')(doc)
+      return markup(doc)
     })
 
   return promise
 }
 
 let projects = ['ember', 'ember-data']
+let releaseToGenDocFor = process.argv[2] ? process.argv[2] : ''
 
-RSVP.map(projects, fetchProject).then(docs => {
-  let giantDocument = {
-    data: _.flatten(docs.map(doc => doc.data))
-  }
-  console.log('normalizing dependencies')
-  normalizeEmberDependencies(giantDocument)
-
-  return putClassesInCouch(giantDocument, db)
-}).then(function () {
-  let glob = require('glob')
+console.log('downloading docs for ' + projects.join(' & '))
 
-  let docs = glob.sync('tmp/docs/**/*.json')
+fetch(db, releaseToGenDocFor).then(downloadedFiles => {
+  RSVP.map(projects, transformProjectFiles).then(docs => {
+    let giantDocument = {
+      data: _.flatten(docs.map(doc => doc.data))
+    }
+    console.log('normalizing dependencies')
+    normalizeEmberDependencies(giantDocument)
 
-  let batchUpdate = require('./lib/batch-update')
+    return putClassesInCouch(giantDocument, db)
+  }).then(function () {
+    let docs = glob.sync('tmp/docs/**/*.json')
 
-  console.log('putting document in CouchDB')
-  return batchUpdate(db, docs)
-}).catch(function (err) {
-  console.warn('err!', err, err.stack)
-  process.exit(1)
+    console.log('putting document in CouchDB')
+    return batchUpdate(db, docs)
+  }).catch(function (err) {
+    console.warn('err!', err, err.stack)
+    process.exit(1)
+  })
 })

lib/batch-update.js

@@ -1,14 +1,14 @@
 'use strict'
 
-let _ = require('lodash')
+let chunk = require('lodash/array').chunk
 let RSVP = require('rsvp')
 let path = require('path')
 let Queue = require('promise-queue')
 let promiseRetry = require('promise-retry')
 
 module.exports = function (db, docs) {
   let queue = new Queue(10)
-  let chunks = _.chunk(docs, 20)
+  let chunks = chunk(docs, 20)
 
   return RSVP.map(chunks, chunk => {
     return promiseRetry((retry, number) => {
@@ -23,7 +23,13 @@ module.exports = function (db, docs) {
 
       if (number > 0) {
         let timesLeft = 10 - number
-        console.log('syncing for chunk failed, retrying ' + timesLeft + ' more times.', keys)
+        if (number !== 1) {
+          if (number !== 0) {
+            console.log('syncing for chunk failed, retrying ' + timesLeft + ' more times.', keys)
+          } else {
+            console.log('syncing for chunk failed, retrying for the last time', keys)
+          }
+        }
       }
 
       return db.allDocs({keys: keys}).catch(retry).then(results => {
@@ -39,7 +45,7 @@ module.exports = function (db, docs) {
             return new RSVP.Promise((resolve, reject) => {
               setTimeout(() => {
                 return db.bulkDocs(docs).then(resolve, reject)
-              }, 1000)
+              }, 3000)
             })
           }).then(() => {
           }).catch(err => {

lib/create-version-index.js

@@ -2,6 +2,9 @@
 
 let saveDoc = require('./save-document')
 let byType = require('./filter-jsonapi-doc').byType
+let fs = require('fs')
+let path = require('path')
+let _ = require('lodash')
 
 // creates a version index at /projects/:project
 // that contains a list of versions in the relationships
@@ -27,5 +30,12 @@ module.exports = function createVersionIndex (db, projectName, versions) {
     }
   }
 
+  let cachedProjectFile = path.join('tmp', 'cached-docs', `project-${projectName}.json`)
+  if (fs.existsSync(cachedProjectFile)) {
+    let cachedDocument = JSON.parse(fs.readFileSync(cachedProjectFile))
+
+    document.data.relationships['project-versions'].data = _.uniq(document.data.relationships['project-versions'].data.concat(cachedDocument.data.relationships['project-versions'].data))
+  }
+
   return saveDoc(document, db).then(() => versions)
 }

lib/fetch.js

@@ -1,7 +1,8 @@
-var S3 = require('s3')
-var RSVP = require('rsvp')
-var path = require('path')
-var fs = require('fs')
+let S3 = require('s3')
+let RSVP = require('rsvp')
+let path = require('path')
+let fs = require('fs')
+let getNewDocsToIndex = require('./identify-docs-to-index')
 
 function mkdirp (dir) {
   return new RSVP.Promise(function (resolve, reject) {
@@ -56,7 +57,7 @@ function downloadFile (document) {
   return mkdirp(path.dirname(finalFile)).then(function () {
     return new RSVP.Promise(function (resolve, reject) {
       if (fs.existsSync(finalFile)) {
-        return resolve()
+        return resolve(finalFile)
       } else {
         client.downloadFile({
           localFile: finalFile,
@@ -66,7 +67,7 @@ function downloadFile (document) {
           }
         })
           .on('end', function () {
-            resolve()
+            resolve(finalFile)
           })
           .on('error', function (err) {
             console.warn('err! ' + err)
@@ -84,6 +85,9 @@ function filterReleaseDocs (document) {
   return versionRegex.test(tag) && /-docs\.json/.test(key)
 }
 
-module.exports = function fetch () {
-  return getObjects().then(downloadReleaseDocs)
+module.exports = function fetch (db, releaseToGenDocFor) {
+  return getObjects().then(docs => {
+    let filteredDocs = docs.filter(doc => doc.Key.indexOf(releaseToGenDocFor) !== -1)
+    return getNewDocsToIndex(db, filteredDocs)
+  }).then(downloadReleaseDocs)
 }

lib/identify-docs-to-index.js

@@ -0,0 +1,52 @@
+let path = require('path')
+let fs = require('fs')
+let RSVP = require('rsvp')
+
+module.exports = function getNewDocsToIndex (db, docs) {
+  return db.allDocs({
+    keys: ['project-ember', 'project-ember-data'],
+    include_docs: true
+  }).then(projectVersions => {
+    let docsToDownload = docs
+
+    if (projectVersions.total_rows > 0) {
+      let [ {doc: emberProjectDoc}, {doc: emberDataProjectDoc} ] = projectVersions.rows
+
+      // Save now for merging later
+      let cachedDocsFolder = path.join('tmp', 'cached-docs')
+      require('mkdirp').sync(cachedDocsFolder)
+      fs.writeFileSync(path.join(cachedDocsFolder, 'project-ember.json'), JSON.stringify(emberProjectDoc))
+      fs.writeFileSync(path.join(cachedDocsFolder, 'project-ember-data.json'), JSON.stringify(emberDataProjectDoc))
+
+      let indexedEmberVersions = []
+      let indexedEmberDataVersions = []
+
+      emberProjectDoc.data.relationships['project-versions'].data.forEach(doc => {
+        indexedEmberVersions.push(doc.id.replace('ember-', ''))
+      })
+
+      emberDataProjectDoc.data.relationships['project-versions'].data.forEach(doc => {
+        indexedEmberDataVersions.push(doc.id.replace('ember-data-', ''))
+      })
+
+      docsToDownload = docsToDownload.filter(doc => {
+        let docString = doc.Key.split('/')
+        let projectName = docString[2].replace('-docs.json', '')
+        let projectVersion = docString[1].replace('v', '')
+
+        if (projectName === 'ember') {
+          return indexedEmberVersions.indexOf(projectVersion) === -1
+        } else if (projectName === 'ember-data') {
+          return indexedEmberDataVersions.indexOf(projectVersion) === -1
+        }
+      })
+
+      if (docsToDownload.length === 0) {
+        console.log('No more docs to index')
+        process.exit(0)
+      }
+    }
+
+    return RSVP.resolve(docsToDownload)
+  })
+}

lib/markup.js

@@ -18,6 +18,8 @@ marked.setOptions({
 
 module.exports = (doc) => {
   doc.data.forEach(document => {
+    console.log('Generating markup for ', document.id)
+
     let description = document.attributes.description
 
     if (description) {

package.json

@@ -4,32 +4,28 @@
   "description": "audits ember docs and generates since tags",
   "main": "index.js",
   "scripts": {
+    "autoformat": "standard --fix index.js lib/*.js",
+    "start": "node index.js",
     "test": "standard && mocha test"
   },
   "author": "Stanley Stuart <stanley+npm@stan.li>",
   "license": "MIT",
   "devDependencies": {
-    "babel": "^6.3.26",
-    "babel-cli": "^6.3.17",
-    "babel-core": "^6.3.26",
-    "babel-preset-es2015": "^6.3.13",
     "chai": "^3.4.1",
-    "mocha": "^2.3.4",
-    "standard": "^5.4.1",
+    "mocha": "^3.2.0",
+    "standard": "^8.6.0",
     "standard-format": "^2.1.0"
   },
   "dependencies": {
-    "cheerio": "^0.19.0",
-    "continuation-local-storage": "^3.1.4",
-    "glob": "^6.0.2",
+    "cheerio": "^0.22.0",
+    "glob": "^7.1.1",
     "graceful-fs": "^4.1.2",
     "highlight.js": "^9.1.0",
     "inflected": "^1.1.6",
     "lodash": "^3.10.1",
-    "lodash.sortby": "^3.1.5",
     "marked": "^0.3.5",
     "mkdirp": "^0.5.1",
-    "pouchdb": "^5.1.0",
+    "pouchdb": "^6.0.7",
     "promise-queue": "^2.2.0",
     "promise-retry": "^1.0.2",
     "rimraf": "^2.5.0",