feat(scripts): Add a bulk mailer

package.json

@@ -65,6 +65,7 @@
     "grunt-nsp": "2.1.2",
     "hawk": "2.3.1",
     "jws": "3.0.0",
+    "leftpad": "0.0.0",
     "load-grunt-tasks": "3.1.0",
     "mailparser": "0.5.1",
     "nock": "1.7.1",

scripts/bulk-mailer.js

@@ -0,0 +1,294 @@
+#!/usr/bin/env node
+
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var commandLineOptions = require('commander')
+var config = require('../config').getProperties()
+var fs = require('fs')
+var leftpad = require('leftpad')
+var log = require('../lib/log')(config.log.level, 'bulk-mailer')
+var Mailer = require('fxa-auth-mailer')
+var nodeMailerMock = require('./bulk-mailer/nodemailer-mock')
+var P = require('bluebird')
+var path = require('path')
+
+commandLineOptions
+  .option('-b, --batchsize [size]', 'Number of emails to send in a batch. Defaults to 10', parseInt)
+  .option('-d, --delay [seconds]', 'Delay in seconds between batches. Defaults to 5', parseInt)
+  .option('-e, --errors <filename>', 'JSON output file that contains errored emails')
+  .option('-i, --input <filename>', 'JSON input file')
+  .option('-t, --template <template>', 'Template filename to render')
+  .option('-u, --unsent <filename>', 'JSON output file that contains emails that were not sent')
+  .option('-w, --write [directory]', 'Directory where emails should be stored')
+  .option('--real', 'Use real email addresses, fake ones are used by default')
+  .option('--send', 'Send emails, for real. *** THIS REALLY SENDS ***')
+  .parse(process.argv)
+
+var BATCH_DELAY = typeof commandLineOptions.delay === "undefined" ? 5 : commandLineOptions.delay
+var BATCH_SIZE = commandLineOptions.batchsize || 10
+
+var requiredOptions = [
+  'errors',
+  'input',
+  'template',
+  'unsent'
+]
+
+requiredOptions.forEach(checkRequiredOption)
+
+var mailer;
+var mailerFunctionName = templateToMailerFunctionName(commandLineOptions.template)
+
+var currentBatch = []
+var emailQueue = []
+var errorCount = 0
+var runningError
+var successCount = 0
+
+P.resolve()
+  .then(createMailer)
+  .then(function (_mailer) {
+    mailer = _mailer
+
+    if (! mailer[mailerFunctionName]) {
+      console.error(commandLineOptions.template, 'is not a valid template')
+      process.exit(1)
+    }
+  })
+  .then(readRecords)
+  .then(normalizeRecords)
+  .then(function (normalizedRecords) {
+    emailQueue = normalizedRecords
+
+    log.info({
+      op: 'send.begin',
+      count: emailQueue.length,
+      test: ! commandLineOptions.send
+    })
+  })
+  .then(nextBatch)
+  .then(function () {
+    log.info({
+      op: 'send.complete',
+      count: errorCount + successCount,
+      successCount: successCount,
+      errorCount: errorCount
+    })
+  })
+  .then(null, function (error) {
+    console.error('error', error)
+    log.error({
+      op: 'send.abort',
+      err: error
+    })
+    runningError = error
+  })
+  .then(writeErrors)
+  .then(writeUnsent)
+  .then(function () {
+    console.log('bye bye!')
+    process.exit(runningError ? 1 : 0)
+  })
+
+var fakeEmailCount = 0
+function normalizeRecords(records) {
+  return records.filter(function (record) {
+    // no email can be sent if the record does not contain an email
+    return !! record.email
+  }).map(function (record) {
+    // real emails are replaced by fake emails by default.
+    if (! commandLineOptions.real) {
+      record.email = 'fake_email' + fakeEmailCount + '@fakedomain.com'
+      fakeEmailCount++
+    }
+
+    // The Chinese translations were handed to us as "zh" w/o a country
+    // specified. We put these translations into "zh-cn", use "zh-cn" for
+    // Taiwan as well.
+    if (! record.acceptLanguage && record.locale) {
+      record.acceptLanguage = record.locale.replace(/zh-tw/gi, 'zh-cn')
+    }
+
+    if (! record.locations) {
+      record.locations = []
+    } else {
+      var translator = mailer.translator(record.acceptLanguage)
+      var language = translator.language
+
+      record.locations.forEach(function (location) {
+        var timestamp = new Date(location.timestamp || location.date)
+        location.timestamp = formatTimestamp(timestamp, record.acceptLanguage)
+
+        // first, try to generate a localized locality
+        if (! location.location && location.citynames && location.countrynames) {
+          var parts = [];
+
+          var localizedCityName = location.citynames[language]
+          if (localizedCityName) {
+            parts.push(localizedCityName)
+          }
+
+          var localizedCountryName = location.countrynames[language]
+          if (localizedCountryName) {
+            parts.push(localizedCountryName)
+          }
+
+          location.location = parts.join(', ')
+        }
+
+        // if that can't be done, fall back to the english locality
+        if (! location.location && location.locality) {
+          location.location = location.locality
+        }
+      })
+    }
+
+    return record
+  })
+}
+
+function nextBatch() {
+  currentBatch = emailQueue.splice(0, BATCH_SIZE)
+
+  if (! currentBatch.length) {
+    return
+  }
+
+  return sendBatch(currentBatch)
+    .then(function () {
+      if (emailQueue.length) {
+        return P.delay(BATCH_DELAY * 1000)
+          .then(nextBatch)
+      }
+    })
+}
+
+function sendBatch(batch) {
+  return P.all(
+    batch.map(function (emailConfig) {
+      return mailer[mailerFunctionName](emailConfig)
+        .then(function () {
+          successCount++
+          log.info({
+            op: 'send.success',
+            email: emailConfig.email
+          })
+        }, function (err) {
+          handleEmailError(emailConfig, err)
+        })
+    })
+  )
+}
+
+var erroredEmailConfigs = []
+function handleEmailError(emailConfig, error) {
+  errorCount++
+
+  emailConfig.error = String(error)
+  erroredEmailConfigs.push(emailConfig)
+
+  log.error({
+    op: 'send.error',
+    email: emailConfig.email,
+    error: error
+  })
+}
+
+// output format should be identical to the input format. This makes it
+// possible to use the error output as input to another test run.
+function writeErrors() {
+  var outputFileName = path.resolve(commandLineOptions.errors)
+  fs.writeFileSync(outputFileName, JSON.stringify(cleanEmailConfigsConfigs(erroredEmailConfigs), null, 2))
+}
+
+function writeUnsent() {
+  var outputFileName = path.resolve(commandLineOptions.unsent)
+
+  // consider all emails in the current batch +
+  // all emails in the emailQueue as unsent.
+  // If there was an error sending the current batch,
+  // we aren't fully sure which are sent, and which aren't.
+  var unsentEmails = [].concat(currentBatch).concat(emailQueue)
+
+  fs.writeFileSync(outputFileName, JSON.stringify(cleanEmailConfigsConfigs(unsentEmails), null, 2))
+}
+
+function cleanEmailConfigsConfigs(erroredEmailConfigs) {
+  return erroredEmailConfigs.map(function (emailConfig) {
+    emailConfig.locations.forEach(function (location) {
+      delete location.translator
+    })
+    return emailConfig
+  })
+}
+
+function camelize(str) {
+  return str.replace(/_(.)/g,
+    function(match, c) {
+      return c.toUpperCase()
+    }
+  )
+}
+
+function templateToMailerFunctionName(templateName) {
+  return camelize(templateName) + 'Email'
+}
+
+function createMailer () {
+  var sender = commandLineOptions.send ? null : nodeMailerMock({
+    failureRate: 0.01,
+    outputDir: commandLineOptions.write ? path.resolve(commandLineOptions.write) : null
+  })
+
+  var defaultLanguage = config.i18n.defaultLanguage
+
+  return Mailer(log, {
+    locales: config.i18n.supportedLanguages,
+    defaultLanguage: defaultLanguage,
+    mail: config.smtp
+  }, sender)
+}
+
+function checkRequiredOption(optionName) {
+  if (! commandLineOptions[optionName]) {
+    console.error('--' + optionName + ' required')
+    process.exit(1)
+  }
+}
+
+function readRecords() {
+  var inputFileName = path.resolve(commandLineOptions.input)
+  var fsStats
+  try {
+    fsStats = fs.statSync(inputFileName)
+  } catch (e) {
+    console.error(inputFileName, 'invalid filename')
+    process.exit(1)
+  }
+
+  if (! fsStats.isFile()) {
+    console.error(inputFileName, 'is not a file')
+    process.exit(1)
+  }
+
+  var records = []
+  try {
+    records = require(inputFileName)
+  } catch(e) {
+    console.error(inputFileName, 'does not contain JSON')
+    process.exit(1)
+  }
+
+  if (! records.length) {
+    console.error('uh oh, no emails found')
+    process.exit(1)
+  }
+
+  return records
+}
+
+function formatTimestamp(timestamp, locale) {
+  return timestamp.getUTCFullYear() + '-' + leftpad(timestamp.getUTCMonth(), 2) + '-' + leftpad(timestamp.getUTCDate(), 2) + ' @ ' + leftpad(timestamp.getUTCHours(), 2) + ':' + leftpad(timestamp.getUTCMinutes(), 2) + ' UTC'
+}

scripts/bulk-mailer/nodemailer-mock.js

@@ -0,0 +1,54 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var fs = require('fs')
+var path = require('path')
+
+module.exports = function (config) {
+  var messageId = 0;
+
+  function ensureOutputDirExists(outputDir) {
+    var dirStats
+    try {
+      dirStats = fs.statSync(config.outputDir)
+    } catch (e) {
+      fs.mkdirSync(outputDir);
+      return;
+    }
+
+    if (! dirStats.isDirectory()) {
+      console.error(outputDir + ' is not a directory');
+      process.exit(1)
+    }
+  }
+
+  return {
+    sendMail: function (emailConfig, callback) {
+      if (config.outputDir) {
+
+        ensureOutputDirExists(config.outputDir)
+
+        var outputPath = path.join(config.outputDir, emailConfig.to)
+
+        var textPath = outputPath + '.txt';
+        fs.writeFileSync(textPath, emailConfig.text)
+
+        var htmlPath = outputPath + '.html'
+        fs.writeFileSync(htmlPath, emailConfig.html)
+      }
+
+      if (Math.random() > config.failureRate) {
+        messageId++
+        callback(null, {
+          message: 'good',
+          messageId: messageId
+        })
+      } else {
+        callback(new Error('uh oh'))
+      }
+    },
+
+    close: function () {}
+  };
+};