fix: able to read feeds with newer items listed last

fixes #52

Author: fent

lib/feedsub.js

@@ -34,6 +34,7 @@ module.exports = class FeedReader extends EventEmitter {
     // Create news emitter.
     this.news = new NewsEmitter({
       maxHistory: this.options.maxHistory,
+      manageHistory: true,
       identifier: (item) => {
         item = item[0];
         return [
@@ -47,17 +48,7 @@ module.exports = class FeedReader extends EventEmitter {
     });
 
     this.news.addHistory('item', this.options.history);
-
-    this.newitems = [];
     this.first = this.news.history.get('item').size === 0;
-
-    // Keep track of and emit new items.
-    this.news.on('item', (item) => {
-      if (!this.first || this.options.emitOnStart) {
-        this.newitems.unshift(item);
-      }
-    });
-
     this.getOpts = Object.assign({
       headers: {},
       acceptEncoding: {
@@ -108,10 +99,12 @@ module.exports = class FeedReader extends EventEmitter {
     let ended = false;
     let aborted = false;
     let req;
+    let items = [];
+    let newItems = [];
+    let sortOrder = 0;
 
     const error = (abort, err) => {
       ended = true;
-      this.newitems = [];
       this.first = false;
       if (!aborted) {
         if (typeof callback === 'function') {
@@ -126,22 +119,21 @@ module.exports = class FeedReader extends EventEmitter {
       }
     };
 
-    const success = (results, abort) => {
+    const success = (abort) => {
+      if (ended) { return; }
       ended = true;
-      this.newitems = [];
+      this.news.addHistory('item', newItems.map((item) => [item]));
+      if (this.first && !this.options.emitOnStart) {
+        newItems = [];
+      }
+      newItems.forEach(this.emit.bind(this, 'item'));
+      this.emit('items', newItems);
       if (abort && !aborted) {
         aborted = true;
-        if (req) {
-          req.abort();
-        }
+        req.abort();
       }
-
-      results.forEach((item) => {
-        this.emit('item', item);
-      });
-      this.emit('items', results);
       if (typeof callback === 'function') {
-        callback(null, results);
+        callback(null, newItems);
       }
     };
 
@@ -164,15 +156,15 @@ module.exports = class FeedReader extends EventEmitter {
     };
 
     if (shouldSkip()) {
-      return success([], true);
+      return success();
     }
 
     req = miniget(this.feed, this.getOpts);
     req.on('response', (res) => {
       // Check if not modified code is sent back
       // in this case, the body will be empty.
       if (res.statusCode === 304) {
-        return success([]);
+        return success();
       }
 
       // Check headers for conditional get.
@@ -231,47 +223,78 @@ module.exports = class FeedReader extends EventEmitter {
       const firstitem = (item) => {
         // If date is the same as last, abort.
         if (date && this.options.lastDate === date) {
-          return success([], true);
+          return success();
         }
 
         if (shouldSkip()) {
-          return success([], true);
+          return success();
         }
 
         // Continue if dates differ.
         if (date) {
           this.options.lastDate = date;
         }
-        parser.on('item', getitems);
-        getitems(item);
+        parser.on('item', getItem);
+        getItem(item);
       };
 
       parser.once('item', firstitem);
 
-      const getitems = (item) => {
-        if (this.first && this.options.emitOnStart) {
-          this.newitems.unshift(item);
+      const getItemDate = (item) => new Date(item.pubdate || item.published || 0);
+      const getItem = (item) => {
+        if (sortOrder === 0) {
+          items.push(item);
+          sortOrder = getItemDate(item) - getItemDate(items[0]);
+          if (sortOrder < 0) {
+            items.forEach(getOlderItem);
+          } else if (sortOrder > 0) {
+            items.forEach(getNewerItem);
+          }
+        } else if (sortOrder < 0) {
+          getOlderItem(item);
+
+        } else {
+          getNewerItem(item);
+        }
+      };
 
-        } else if (!this.news.emit('item', item)) {
-          // Check if this item has already been read in previous requests
-          // if it has, then stop parsing the rest of the document.
-          parser.removeListener('item', getitems);
-          success(this.newitems);
+      const getOlderItem = (item) => {
+        if (this.first) {
+          newItems.unshift(item);
+        } else if (!ended) {
+          let emitted = this.news.emit('item', item);
+          if (emitted) {
+            newItems.unshift(item);
+          } else {
+            // Check if this item has already been read in previous requests
+            // if it has, then stop parsing the rest of the document.
+            parser.removeListener('item', getItem);
+            success(true);
+          }
+        }
+      };
+
+      let foundPrevItem = false;
+      const getNewerItem = (item) => {
+        if (this.first) {
+          newItems.push(item);
+        } else if (!foundPrevItem && !this.news.emit('item', item)) {
+          foundPrevItem = true;
+        } else if (foundPrevItem && this.news.emit('item', item)) {
+          newItems.push(item);
         }
       };
 
       req.pipe(parser);
 
       req.on('end', () => {
-        if (!ended) {
-          if (this.first && this.options.emitOnStart) {
-            this.news.addHistory('item', this.newitems.map((item) => {
-              return [item];
-            }));
-          }
-          success(this.newitems);
-          this.first = false;
+        if (ended) { return; }
+        // Process items in descending order if no order found at end.
+        if (sortOrder === 0 && newItems.length === 0) {
+          items.forEach(getOlderItem);
         }
+        success();
+        this.first = false;
       });
     });