feat: add explain support (#2626)

Explain support for specific commands is accessible via a
new `explain` option, either a boolean or a string specifying
the requested verbosity, at the operation level. Explainable
cursor operations can also be explained via the existing
cursor `explain` method, which now takes an optional verbosity 
parameter (defaults to true for backwards compatibility).

NODE-2853

lib/aggregation_cursor.js

@@ -322,7 +322,13 @@ AggregationCursor.prototype.get = AggregationCursor.prototype.toArray;
 
 /**
  * Execute the explain for the cursor
+ *
+ * For backwards compatibility, a verbosity of true is interpreted as "allPlansExecution"
+ * and false as "queryPlanner". Prior to server version 3.6, aggregate()
+ * ignores the verbosity parameter and executes in "queryPlanner".
+ *
  * @method AggregationCursor.prototype.explain
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [verbosity=true] - An optional mode in which to run the explain.
  * @param {AggregationCursor~resultCallback} [callback] The result callback.
  * @return {Promise} returns Promise if no callback passed
  */

lib/collection.js

@@ -287,7 +287,6 @@ const DEPRECATED_FIND_OPTIONS = ['maxScan', 'fields', 'snapshot', 'oplogReplay']
  * @param {object} [options.fields] **Deprecated** Use `options.projection` instead
  * @param {number} [options.skip=0] Set to skip N documents ahead in your query (useful for pagination).
  * @param {Object} [options.hint] Tell the query to use specific indexes in the query. Object of indexes to use, {'_id':1}
- * @param {boolean} [options.explain=false] Explain the query instead of returning the data.
  * @param {boolean} [options.snapshot=false] DEPRECATED: Snapshot query.
  * @param {boolean} [options.timeout=false] Specify if the cursor can timeout.
  * @param {boolean} [options.tailable=false] Specify if the cursor is tailable.
@@ -310,6 +309,7 @@ const DEPRECATED_FIND_OPTIONS = ['maxScan', 'fields', 'snapshot', 'oplogReplay']
  * @param {boolean} [options.noCursorTimeout] The server normally times out idle cursors after an inactivity period (10 minutes) to prevent excess memory use. Set this option to prevent that.
  * @param {object} [options.collation] Specify collation (MongoDB 3.4 or higher) settings for update operation (see 3.4 documentation for available fields).
  * @param {boolean} [options.allowDiskUse] Enables writing to temporary files on the server.
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @throws {MongoError}
  * @return {Cursor}
@@ -744,6 +744,7 @@ Collection.prototype.insert = deprecate(function(docs, options, callback) {
  * @param {boolean} [options.checkKeys=false] If true, will throw if bson documents start with `$` or include a `.` in any key value
  * @param {boolean} [options.serializeFunctions=false] Serialize functions on any object.
  * @param {boolean} [options.ignoreUndefined=false] Specify if the BSON serializer should ignore undefined fields.
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @param {Collection~updateWriteOpCallback} [callback] The command result callback
  * @return {Promise} returns Promise if no callback passed
@@ -821,6 +822,7 @@ Collection.prototype.replaceOne = function(filter, doc, options, callback) {
  * @param {boolean} [options.checkKeys=false] If true, will throw if bson documents start with `$` or include a `.` in any key value
  * @param {boolean} [options.serializeFunctions=false] Serialize functions on any object.
  * @param {boolean} [options.ignoreUndefined=false] Specify if the BSON serializer should ignore undefined fields.
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @param {Collection~updateWriteOpCallback} [callback] The command result callback
  * @return {Promise<Collection~updateWriteOpResult>} returns Promise if no callback passed
@@ -912,6 +914,7 @@ Collection.prototype.update = deprecate(function(selector, update, options, call
  * @param {boolean} [options.checkKeys=false] If true, will throw if bson documents start with `$` or include a `.` in any key value
  * @param {boolean} [options.serializeFunctions=false] Serialize functions on any object.
  * @param {boolean} [options.ignoreUndefined=false] Specify if the BSON serializer should ignore undefined fields.
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @param {string|object} [options.hint] optional index hint for optimizing the filter query
  * @param {Collection~deleteWriteOpCallback} [callback] The command result callback
@@ -947,6 +950,7 @@ Collection.prototype.removeOne = Collection.prototype.deleteOne;
  * @param {boolean} [options.checkKeys=false] If true, will throw if bson documents start with `$` or include a `.` in any key value
  * @param {boolean} [options.serializeFunctions=false] Serialize functions on any object.
  * @param {boolean} [options.ignoreUndefined=false] Specify if the BSON serializer should ignore undefined fields.
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @param {string|object} [options.hint] optional index hint for optimizing the filter query
  * @param {Collection~deleteWriteOpCallback} [callback] The command result callback
@@ -1056,7 +1060,6 @@ Collection.prototype.save = deprecate(function(doc, options, callback) {
  * @param {object} [options.fields] **Deprecated** Use `options.projection` instead
  * @param {number} [options.skip=0] Set to skip N documents ahead in your query (useful for pagination).
  * @param {Object} [options.hint] Tell the query to use specific indexes in the query. Object of indexes to use, {'_id':1}
- * @param {boolean} [options.explain=false] Explain the query instead of returning the data.
  * @param {boolean} [options.snapshot=false] DEPRECATED: Snapshot query.
  * @param {boolean} [options.timeout=false] Specify if the cursor can timeout.
  * @param {boolean} [options.tailable=false] Specify if the cursor is tailable.
@@ -1075,6 +1078,7 @@ Collection.prototype.save = deprecate(function(doc, options, callback) {
  * @param {boolean} [options.partial=false] Specify if the cursor should return partial results when querying against a sharded system
  * @param {number} [options.maxTimeMS] Number of milliseconds to wait before aborting the query.
  * @param {object} [options.collation] Specify collation (MongoDB 3.4 or higher) settings for update operation (see 3.4 documentation for available fields).
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @param {Collection~resultCallback} [callback] The command result callback
  * @return {Promise} returns Promise if no callback passed
@@ -1595,6 +1599,7 @@ Collection.prototype.countDocuments = function(query, options, callback) {
  * @param {(ReadPreference|string)} [options.readPreference] The preferred read preference (ReadPreference.PRIMARY, ReadPreference.PRIMARY_PREFERRED, ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED, ReadPreference.NEAREST).
  * @param {number} [options.maxTimeMS] Number of milliseconds to wait before aborting the query.
  * @param {object} [options.collation] Specify collation settings for operation. See {@link https://docs.mongodb.com/manual/reference/command/aggregate|aggregation documentation}.
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @param {Collection~resultCallback} [callback] The command result callback
  * @return {Promise} returns Promise if no callback passed
@@ -1674,6 +1679,7 @@ Collection.prototype.stats = function(options, callback) {
  * @param {boolean} [options.checkKeys=false] If true, will throw if bson documents start with `$` or include a `.` in any key value
  * @param {boolean} [options.serializeFunctions=false] Serialize functions on any object.
  * @param {boolean} [options.ignoreUndefined=false] Specify if the BSON serializer should ignore undefined fields.
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @param {Collection~findAndModifyCallback} [callback] The collection result callback
  * @return {Promise<Collection~findAndModifyWriteOpResultObject>} returns Promise if no callback passed
@@ -1707,6 +1713,7 @@ Collection.prototype.findOneAndDelete = function(filter, options, callback) {
  * @param {boolean} [options.checkKeys=false] If true, will throw if bson documents start with `$` or include a `.` in any key value
  * @param {boolean} [options.serializeFunctions=false] Serialize functions on any object.
  * @param {boolean} [options.ignoreUndefined=false] Specify if the BSON serializer should ignore undefined fields.
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @param {Collection~findAndModifyCallback} [callback] The collection result callback
  * @return {Promise<Collection~findAndModifyWriteOpResultObject>} returns Promise if no callback passed
@@ -1741,6 +1748,7 @@ Collection.prototype.findOneAndReplace = function(filter, replacement, options,
  * @param {boolean} [options.checkKeys=false] If true, will throw if bson documents start with `$` or include a `.` in any key value
  * @param {boolean} [options.serializeFunctions=false] Serialize functions on any object.
  * @param {boolean} [options.ignoreUndefined=false] Specify if the BSON serializer should ignore undefined fields.
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] An ptional session to use for this operation
  * @param {Collection~findAndModifyCallback} [callback] The collection result callback
  * @return {Promise<Collection~findAndModifyWriteOpResultObject>} returns Promise if no callback passed
@@ -1848,7 +1856,6 @@ Collection.prototype.findAndRemove = deprecate(function(query, sort, options, ca
  * @param {number} [options.batchSize=1000] The number of documents to return per batch. See {@link https://docs.mongodb.com/manual/reference/command/aggregate|aggregation documentation}.
  * @param {object} [options.cursor] Return the query as cursor, on 2.6 > it returns as a real cursor on pre 2.6 it returns as an emulated cursor.
  * @param {number} [options.cursor.batchSize=1000] Deprecated. Use `options.batchSize`
- * @param {boolean} [options.explain=false] Explain returns the aggregation execution plan (requires mongodb 2.6 >).
  * @param {boolean} [options.allowDiskUse=false] allowDiskUse lets the server know if it can use disk to store temporary results for the aggregation (requires mongodb 2.6 >).
  * @param {number} [options.maxTimeMS] maxTimeMS specifies a cumulative time limit in milliseconds for processing operations on the cursor. MongoDB interrupts the operation at the earliest following interrupt point.
  * @param {number} [options.maxAwaitTimeMS] The maximum amount of time for the server to wait on new documents to satisfy a tailable cursor query.
@@ -1860,6 +1867,7 @@ Collection.prototype.findAndRemove = deprecate(function(query, sort, options, ca
  * @param {object} [options.collation] Specify collation settings for operation. See {@link https://docs.mongodb.com/manual/reference/command/aggregate|aggregation documentation}.
  * @param {string} [options.comment] Add a comment to an aggregation command
  * @param {string|object} [options.hint] Add an index selection hint to an aggregation command
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @param {Collection~aggregationCallback} callback The command result callback
  * @return {(null|AggregationCursor)}
@@ -2106,6 +2114,7 @@ Collection.prototype.group = deprecate(function(
  * @param {boolean} [options.jsMode=false] It is possible to make the execution stay in JS. Provided in MongoDB > 2.0.X.
  * @param {boolean} [options.verbose=false] Provide statistics on job execution time.
  * @param {boolean} [options.bypassDocumentValidation=false] Allow driver to bypass schema validation in MongoDB 3.2 or higher.
+ * @param {'queryPlanner'|'queryPlannerExtended'|'executionStats'|'allPlansExecution'|boolean} [options.explain] The verbosity mode for the explain output.
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @param {Collection~resultCallback} [callback] The command result callback
  * @throws {MongoError}

lib/core/sdam/topology.js

@@ -930,7 +930,8 @@ function executeWriteOperation(args, options, callback) {
     !!options.retryWrites &&
     options.session &&
     isRetryableWritesSupported(topology) &&
-    !options.session.inTransaction();
+    !options.session.inTransaction() &&
+    options.explain === undefined;
 
   topology.selectServer(writableServerSelector(), options, (err, server) => {
     if (err) {

lib/core/topologies/mongos.js

@@ -919,7 +919,8 @@ function executeWriteOperation(args, options, callback) {
     !!options.retryWrites &&
     options.session &&
     isRetryableWritesSupported(self) &&
-    !options.session.inTransaction();
+    !options.session.inTransaction() &&
+    options.explain === undefined;
 
   const handler = (err, result) => {
     if (!err) return callback(null, result);

lib/core/topologies/replset.js

@@ -1193,7 +1193,8 @@ function executeWriteOperation(args, options, callback) {
     !!options.retryWrites &&
     options.session &&
     isRetryableWritesSupported(self) &&
-    !options.session.inTransaction();
+    !options.session.inTransaction() &&
+    options.explain === undefined;
 
   if (!self.s.replicaSetState.hasPrimary()) {
     if (self.s.disconnectHandler) {

lib/core/wireprotocol/query.js

@@ -8,6 +8,8 @@ const isSharded = require('./shared').isSharded;
 const maxWireVersion = require('../utils').maxWireVersion;
 const applyCommonQueryOptions = require('./shared').applyCommonQueryOptions;
 const command = require('./command');
+const decorateWithExplain = require('../../utils').decorateWithExplain;
+const Explain = require('../../explain').Explain;
 
 function query(server, ns, cmd, cursorState, options, callback) {
   options = options || {};
@@ -31,7 +33,14 @@ function query(server, ns, cmd, cursorState, options, callback) {
   }
 
   const readPreference = getReadPreference(cmd, options);
-  const findCmd = prepareFindCommand(server, ns, cmd, cursorState, options);
+  let findCmd = prepareFindCommand(server, ns, cmd, cursorState, options);
+
+  // If we have explain, we need to rewrite the find command
+  // to wrap it in the explain command
+  const explain = Explain.fromOptions(options);
+  if (explain) {
+    findCmd = decorateWithExplain(findCmd, explain);
+  }
 
   // NOTE: This actually modifies the passed in cmd, and our code _depends_ on this
   //       side-effect. Change this ASAP
@@ -59,7 +68,7 @@ function query(server, ns, cmd, cursorState, options, callback) {
 
 function prepareFindCommand(server, ns, cmd, cursorState) {
   cursorState.batchSize = cmd.batchSize || cursorState.batchSize;
-  let findCmd = {
+  const findCmd = {
     find: collectionNamespace(ns)
   };
 
@@ -143,14 +152,6 @@ function prepareFindCommand(server, ns, cmd, cursorState) {
   if (cmd.collation) findCmd.collation = cmd.collation;
   if (cmd.readConcern) findCmd.readConcern = cmd.readConcern;
 
-  // If we have explain, we need to rewrite the find command
-  // to wrap it in the explain command
-  if (cmd.explain) {
-    findCmd = {
-      explain: findCmd
-    };
-  }
-
   return findCmd;
 }
 
@@ -188,7 +189,7 @@ function prepareLegacyFindQuery(server, ns, cmd, cursorState, options) {
   if (typeof cmd.showDiskLoc !== 'undefined') findCmd['$showDiskLoc'] = cmd.showDiskLoc;
   if (cmd.comment) findCmd['$comment'] = cmd.comment;
   if (cmd.maxTimeMS) findCmd['$maxTimeMS'] = cmd.maxTimeMS;
-  if (cmd.explain) {
+  if (options.explain !== undefined) {
     // nToReturn must be 0 (match all) or negative (match N and close cursor)
     // nToReturn > 0 will give explain results equivalent to limit(0)
     numberToReturn = -Math.abs(cmd.limit || 0);

lib/core/wireprotocol/write_command.js

@@ -3,6 +3,8 @@
 const MongoError = require('../error').MongoError;
 const collectionNamespace = require('./shared').collectionNamespace;
 const command = require('./command');
+const decorateWithExplain = require('../../utils').decorateWithExplain;
+const Explain = require('../../explain').Explain;
 
 function writeCommand(server, type, opsField, ns, ops, options, callback) {
   if (ops.length === 0) throw new MongoError(`${type} must contain at least one document`);
@@ -15,7 +17,7 @@ function writeCommand(server, type, opsField, ns, ops, options, callback) {
   const ordered = typeof options.ordered === 'boolean' ? options.ordered : true;
   const writeConcern = options.writeConcern;
 
-  const writeCommand = {};
+  let writeCommand = {};
   writeCommand[type] = collectionNamespace(ns);
   writeCommand[opsField] = ops;
   writeCommand.ordered = ordered;
@@ -36,6 +38,13 @@ function writeCommand(server, type, opsField, ns, ops, options, callback) {
     writeCommand.bypassDocumentValidation = options.bypassDocumentValidation;
   }
 
+  // If a command is to be explained, we need to reformat the command after
+  // the other command properties are specified.
+  const explain = Explain.fromOptions(options);
+  if (explain) {
+    writeCommand = decorateWithExplain(writeCommand, explain);
+  }
+
   const commandOptions = Object.assign(
     {
       checkKeys: type === 'insert',