Merge pull request #134 from kuzzleio/develop-implement-scroll

Implement scroll api

Author: David Bengsch

.eslintignore

@@ -0,0 +1,12 @@
+coverage:
+  range: "90...100"
+
+  status:
+    project:
+      default:
+        threshold: 1
+
+    patch:
+      default:
+        branches:
+          - master

.jshintrc

@@ -20,7 +20,7 @@
   "scripts": {
     "test": "npm run --silent lint && npm run unit-testing",
     "unit-testing": "istanbul cover _mocha  -- --recursive",
-    "lint": "eslint --max-warnings=0 ./lib ./test",
+    "lint": "eslint --max-warnings=0 ./src ./test",
     "build": "node build.js"
   },
   "browser": "src/kuzzle.js",
@@ -38,14 +38,8 @@
     "webpack": "^1.13.1",
     "eslint": "^3.7.1",
     "eslint-friendly-formatter": "^2.0.6",
-    "eslint-loader": "^1.5.0",
     "browserify": "13.1.0",
     "codecov": "^1.0.1",
-    "grunt": "1.0.1",
-    "grunt-browserify": "^5.0.0",
-    "grunt-contrib-uglify": "2.0.0",
-    "grunt-webpack": "^1.0.14",
-    "gruntify-eslint": "^3.0.0",
     "istanbul": "0.4.5",
     "istanbul-middleware": "0.2.2",
     "mocha": "3.1.0",

codecov.yml

@@ -23,7 +23,7 @@ var
  * @param {responseCallback} [cb] - Handles connection response
  * @constructor
  */
-module.exports = Kuzzle = function (host, options, cb) {
+function Kuzzle (host, options, cb) {
   var self = this;
 
   if (!(this instanceof Kuzzle)) {
@@ -309,7 +309,7 @@ module.exports = Kuzzle = function (host, options, cb) {
       }
     });
   }
-};
+}
 
 /**
  * Connects to a Kuzzle instance using the provided host name.
@@ -517,6 +517,32 @@ Kuzzle.prototype.login = function (strategy) {
   });
 };
 
+/**
+ * Create a kuzzle index
+ *
+ * @param {string} index
+ * @param {object} [options]
+ * @param {responseCallback} cb
+ * @returns {Kuzzle}
+ */
+Kuzzle.prototype.createIndex = function (index, options, cb) {
+  if (!index) {
+    if (!this.defaultIndex) {
+      throw new Error('Kuzzle.createIndex: index required');
+    }
+    index = this.defaultIndex;
+  }
+
+  if (!cb && typeof options === 'function') {
+    cb = options;
+    options = null;
+  }
+
+  this.query({controller: 'admin', action: 'createIndex'}, {index: index}, options, typeof cb !== 'function' ? null : cb);
+
+  return this;
+};
+
 /**
  * Send logout request to kuzzle with jwtToken.
  *
@@ -913,7 +939,7 @@ Kuzzle.prototype.getStatistics = function (timestamp, options, cb) {
  *
  * @param {string} collection - The name of the data collection you want to manipulate
  * @param {string} [index] - The name of the data index containing the data collection
- * @returns {object} A KuzzleDataCollection instance
+ * @returns {KuzzleDataCollection} A KuzzleDataCollection instance
  */
 Kuzzle.prototype.dataCollectionFactory = function(collection, index) {
   this.isValid();
@@ -1459,3 +1485,5 @@ Kuzzle.prototype.stopQueuing = function () {
 
   return this;
 };
+
+module.exports = Kuzzle;

package.json

@@ -1,4 +1,5 @@
 var
+  KuzzleSearchResult = require('./kuzzleSearchResult'),
   KuzzleDocument = require('./kuzzleDocument'),
   KuzzleDataMapping = require('./kuzzleDataMapping'),
   KuzzleRoom = require('./kuzzleRoom'),
@@ -15,6 +16,11 @@ var
 /**
  * A data collection is a set of data managed by Kuzzle. It acts like a data table for persistent documents,
  * or like a room for pub/sub messages.
+ *
+ * @property {string} collection
+ * @property {string} index
+ * @property {Kuzzle} kuzzle
+ * @property {Array.<string>} collection
  * @param {object} kuzzle - Kuzzle instance to inherit from
  * @param {string} collection - name of the data collection to handle
  * @param {string} index - Index containing the data collection
@@ -72,61 +78,6 @@ function KuzzleDataCollection(kuzzle, collection, index) {
   return this;
 }
 
-/**
- * Executes an advanced search on the data collection.
- *
- * /!\ There is a small delay between documents creation and their existence in our advanced search layer,
- * usually a couple of seconds.
- * That means that a document that was just been created won’t be returned by this function.
- *
- * @param {object} filters - Filters in Elasticsearch Query DSL format
- * @param {object} [options] - Optional parameters
- * @param {responseCallback} cb - Handles the query response
- */
-KuzzleDataCollection.prototype.advancedSearch = function (filters, options, cb) {
-  var
-    query,
-    self = this;
-
-  if (!cb && typeof options === 'function') {
-    cb = options;
-    options = null;
-  }
-
-  self.kuzzle.callbackRequired('KuzzleDataCollection.advancedSearch', cb);
-
-  query = self.kuzzle.addHeaders({body: filters}, this.headers);
-
-  self.kuzzle.query(this.buildQueryArgs('document', 'search'), query, options, function (error, result) {
-    var
-      response,
-      documents = [];
-
-    if (error) {
-      return cb(error);
-    }
-
-    result.result.hits.forEach(function (doc) {
-      var newDocument = new KuzzleDocument(self, doc._id, doc._source);
-
-      newDocument.version = doc._version;
-
-      documents.push(newDocument);
-    });
-
-    response = {
-      total: result.result.total,
-      documents: documents
-    };
-
-    if (result.result.aggregations) {
-      response.aggregations = result.result.aggregations;
-    }
-
-    cb(null, response);
-  });
-};
-
 /**
  * Returns the number of documents matching the provided set of filters.
  *
@@ -332,16 +283,51 @@ KuzzleDataCollection.prototype.fetchDocument = function (documentId, options, cb
  * @param {responseCallback} cb - Handles the query response
  */
 KuzzleDataCollection.prototype.fetchAllDocuments = function (options, cb) {
-  var filters = {};
+  var
+    warnEmitted = false,
+    documents = [],
+    filters = {};
 
   if (!cb && typeof options === 'function') {
     cb = options;
-    options = null;
+    options = {};
   }
 
-  this.kuzzle.callbackRequired('KuzzleDataCollection.fetchAll', cb);
+  // copying pagination options to the search filter
+  if (!options) {
+    options = {};
+  }
+
+  if (!options.from) {
+    options.from = 0;
+  }
+
+  if (!options.size) {
+    options.size = 1000;
+  }
+
+  this.kuzzle.callbackRequired('KuzzleDataCollection.fetchAllDocuments', cb);
+
+  this.search(filters, options, function getNextDocuments (error, searchResult) {
+    if (error) {
+      return cb(error);
+    }
+
+    if (searchResult instanceof KuzzleSearchResult) {
+      if (searchResult.total > 10000 && !warnEmitted) {
+        warnEmitted = true;
+        console.warn('KuzzleDataCollection.fetchAllDocuments may return extremely large amounts of documents, which may cause performance issues. Unless you know what you are doing, consider using KuzzleDataCollection.search or KuzzleDataCollection.scroll instead'); // eslint-disable-line no-console
+      }
 
-  this.advancedSearch(filters, options, cb);
+      searchResult.documents.forEach(document => {
+        documents.push(document);
+      });
+      searchResult.next(getNextDocuments);
+    }
+    else {
+      cb(null, documents);
+    }
+  });
 };
 
 
@@ -435,6 +421,131 @@ KuzzleDataCollection.prototype.replaceDocument = function (documentId, content,
   return this;
 };
 
+/**
+ * Executes an advanced search on the data collection.
+ *
+ * /!\ There is a small delay between documents creation and their existence in our advanced search layer,
+ * usually a couple of seconds.
+ * That means that a document that was just been created won’t be returned by this function.
+ *
+ * @param {object} filters - Filters in Elasticsearch Query DSL format
+ * @param {object} [options] - Optional parameters
+ * @param {responseCallback} cb - Handles the query response
+ */
+
+KuzzleDataCollection.prototype.search = function (filters, options, cb) {
+  var
+    query,
+    self = this;
+
+  if (!cb && typeof options === 'function') {
+    cb = options;
+    options = {};
+  }
+
+  self.kuzzle.callbackRequired('KuzzleDataCollection.search', cb);
+
+  query = self.kuzzle.addHeaders({body: filters}, this.headers);
+
+
+  self.kuzzle.query(this.buildQueryArgs('document', 'search'), query, options, function (error, result) {
+    var documents = [];
+
+    if (error) {
+      return cb(error);
+    }
+
+    result.result.hits.forEach(function (doc) {
+      var newDocument = new KuzzleDocument(self, doc._id, doc._source);
+
+      newDocument.version = doc._version;
+
+      documents.push(newDocument);
+    });
+
+    if (result.result._scroll_id) {
+      options.scrollId = result.result._scroll_id;
+    }
+
+    cb(null, new KuzzleSearchResult(
+      self,
+      result.result.total,
+      documents,
+      result.result.aggregations ? result.result.aggregations : [],
+      {options: options, filters: filters}
+    ));
+  });
+};
+
+/**
+ * A "scroll" option can be passed to search queries, creating persistent
+ * paginated results.
+ * This method can be used to manually get the next page of a search result,
+ * instead of using KuzzleSearchResult.next()
+ *
+ * @param {string} scrollId
+ * @param {object} [options]
+ * @param {object} [filters]
+ * @param {responseCallback} cb
+ */
+KuzzleDataCollection.prototype.scroll = function (scrollId, options, filters, cb) {
+  var
+    request = {body: {}},
+    self = this;
+
+  if (!scrollId) {
+    throw new Error('KuzzleDataCollection.scroll: scrollId required');
+  }
+
+  if (!cb) {
+    cb = filters;
+    filters = null;
+  }
+
+  if (!cb && typeof options === 'function') {
+    cb = options;
+    options = {};
+  }
+
+  if (!options) {
+    options = {};
+  }
+
+  options.scrollId = scrollId;
+
+  this.kuzzle.callbackRequired('KuzzleDataCollection.scroll', cb);
+
+  this.kuzzle.query({controller: 'document', action: 'scroll'}, request, options, function (error, result) {
+    var documents = [];
+
+    if (error) {
+      return cb(error);
+    }
+
+    result.result.hits.forEach(function (doc) {
+      var newDocument = new KuzzleDocument(self, doc._id, doc._source);
+
+      newDocument.version = doc._version;
+
+      documents.push(newDocument);
+    });
+
+    if (result.result._scroll_id) {
+      options.scrollId = result.result._scroll_id;
+    }
+
+    cb(null, new KuzzleSearchResult(
+      self,
+      result.result.total,
+      documents,
+      result.result.aggregations ? result.result.aggregations : [],
+      {options: options, filters: filters}
+    ));
+  });
+
+  return this;
+};
+
 /**
  * Subscribes to this data collection with a set of filters.
  * To subscribe to the entire data collection, simply provide an empty filter.

src/kuzzle.js

@@ -16,7 +16,7 @@
  *  The KuzzleDataMapping object allow to get the current mapping of a data collection and to modify it if needed.
  *
  * @param {object} kuzzleDataCollection - Instance of the inherited KuzzleDataCollection object
- * @param {object} mapping - mappings
+ * @param {object} [mapping] - mappings
  * @constructor
  */
 function KuzzleDataMapping(kuzzleDataCollection, mapping) {