Merge pull request #74 from linkedconnections/development

v2.1.4

lib/Gtfsrt2LC.js

@@ -112,7 +112,7 @@ class Gtfsrt2LC {
                // Figure service date and trip start time
                let serviceDay = null;
                let tripStartTime = null;
-               
+
                if (tripUpdate.trip.startDate) {
                   const rawStartDate = tripUpdate.trip.startDate;
                   serviceDay = new Date(
@@ -401,21 +401,21 @@ class Gtfsrt2LC {
       );
       const tomorrowServiceDate = addDays(todayServiceDate, 1);
       const yesterdayServiceDate = addDays(todayServiceDate, -1);
-      
+
       const todayDistance = service[today] === '1' ? Math.abs(now - todayServiceDate) : Number.POSITIVE_INFINITY;
       const tomorrowDistance = service[tomorrow] === '1' ? Math.abs(now - tomorrowServiceDate) : Number.POSITIVE_INFINITY;
       const yesterdayDistance = service[yesterday] === '1' ? Math.abs(now - yesterdayServiceDate) : Number.POSITIVE_INFINITY;
 
-      if(todayDistance === Math.min(todayDistance, tomorrowDistance, yesterdayDistance)) {
-         return todayServiceDate.setUTCHours(0, 0, 0 ,0);
+      if (todayDistance === Math.min(todayDistance, tomorrowDistance, yesterdayDistance)) {
+         return todayServiceDate.setUTCHours(0, 0, 0, 0);
       }
 
-      if(tomorrowDistance === Math.min(todayDistance, tomorrowDistance, yesterdayDistance)) {
-         return tomorrowServiceDate.setUTCHours(0, 0, 0 ,0);
+      if (tomorrowDistance === Math.min(todayDistance, tomorrowDistance, yesterdayDistance)) {
+         return tomorrowServiceDate.setUTCHours(0, 0, 0, 0);
       }
 
-      if(yesterdayDistance === Math.min(todayDistance, tomorrowDistance, yesterdayDistance)) {
-         return yesterdayServiceDate.setUTCHours(0, 0, 0 ,0);
+      if (yesterdayDistance === Math.min(todayDistance, tomorrowDistance, yesterdayDistance)) {
+         return yesterdayServiceDate.setUTCHours(0, 0, 0, 0);
       }
    }
 
@@ -557,6 +557,21 @@ class Gtfsrt2LC {
                   update['departure']['time'] = (this.addDuration(serviceDay, this.parseGTFSDuration(staticData['departure_time'])).getTime() / 1000) + update['departure']['delay'];
                }
             }
+         } else {
+            // If this stop is not the last of the trip and the stop update is missing departure info
+            // add it manually taking into account the arrival delay at this stop
+            if (staticIndex < staticLength - 1 && update['arrival']) {
+               update['departure'] = {
+                  'delay': update['arrival']['delay'] | 0,
+                  'time': (this.addDuration(serviceDay, this.parseGTFSDuration(staticData['departure_time'])).getTime() / 1000) + update['arrival']['delay']
+               }
+            } else {
+               // Fallback to static data
+               update['departure'] = {
+                  'delay': 0,
+                  'time': (this.addDuration(serviceDay, this.parseGTFSDuration(staticData['departure_time'])).getTime() / 1000)
+               }
+            }
          }
 
          // Check if arrival time is explicitly defined. In some cases only the delay is given
@@ -570,23 +585,10 @@ class Gtfsrt2LC {
                   update['arrival']['time'] = (this.addDuration(serviceDay, this.parseGTFSDuration(staticData['arrival_time'])).getTime() / 1000) + update['arrival']['delay'];
                }
             }
-         }
-
-         // If this stop is not the last of the trip and the stop update is missing departure info
-         // add it manually taking into account the arrival delay at this stop
-         if (staticIndex != staticLength - 1) {
-            if (!update['departure']) {
-               update['departure'] = {
-                  'delay': update['arrival']['delay'],
-                  'time': (this.addDuration(serviceDay, this.parseGTFSDuration(staticData['departure_time'])).getTime() / 1000) + update['arrival']['delay']
-               }
-            }
-         }
-
-         // If the stop update is missing arrival info and is not the first stop of the trip
-         // add it manually taking into account the departure delay of the previous stop (if any)
-         if (staticIndex != 0) {
-            if (!update['arrival'] && prevUpdate) {
+         } else {
+            // If the stop update is missing arrival info and is not the first stop of the trip
+            // add it manually taking into account the departure delay of the previous stop (if any)
+            if (staticIndex > 0 && prevUpdate) {
                // We need to make sure that adding the departure delay of the previous stop 
                // to the arrival time of this stop won't cause inconsistent times,
                // i.e. arrival > departure.
@@ -625,10 +627,10 @@ class Gtfsrt2LC {
 
                   }
                }
-            }
+            } else { /* This should never happen */ }
          }
 
-         // Check for consistencies between this update and the previous
+         // Check for inconsistencies between this update and the previous
          if (prevUpdate && update['departure'] && prevUpdate['departure']['time'] > update['arrival']['time']) {
             // Enforce previous delay on this update to keep consistency
             let prevDepDelay = prevUpdate ? prevUpdate['departure']['delay'] : 0;
@@ -650,6 +652,8 @@ class Gtfsrt2LC {
          }
       } catch (err) {
          console.error(err);
+         console.error('Issue encountered while processing this update: ', JSON.stringify(update, null, 3));
+         console.error('From this trip: ', JSON.stringify(staticData, null, 3));
       }
 
       return update;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gtfsrt2lc",
-  "version": "2.1.3",
+  "version": "2.1.4",
   "description": "Converts the GTFS-RT to Linked Connections",
   "main": "./Gtfsrt2LC.js",
   "bin": {

test/gtfsrt2lc.test.js

@@ -2,6 +2,7 @@ const fs = require('fs');
 const del = require('del');
 const { Readable } = require('stream');
 const uri_templates = require('uri-templates');
+const { Level } = require('level');
 const GtfsIndex = require('../lib/GtfsIndex');
 const Gtfsrt2lc = require('../lib/Gtfsrt2LC');
 const Utils = require('../lib/Utils');
@@ -114,6 +115,50 @@ test('Extract all indexes when source is given as decompressed folder', async ()
     await del(['./test/data/decompressed'], { force: true });
 });
 
+test('Historic records are used to prune unchanged connections', async () => {
+    expect.assertions(4);
+    const historyDB = new Level('./test/data/history.db', { valueEncoding: 'json' });
+    const gti = new GtfsIndex({ path: static_path });
+    const indexes = await gti.getIndexes({ store: 'MemStore' });
+
+    // First run
+    const grt1 = new Gtfsrt2lc({ 
+        path: rt_path,
+        uris: mock_uris,
+    });
+    grt1.setIndexes({ ...indexes, historyDB });
+    const connStream1 = await grt1.parse({ format: 'jsonld', objectMode: true });
+    let count1 = 0;
+    const endStream = new Promise(res => {
+        connStream1.on('end', () => res(true))
+            .on('error', () => res(false));
+    });
+    connStream1.on('data', conn => { count1++; });
+    const success1 = await endStream;
+
+    // Second run
+    const grt2 = new Gtfsrt2lc({ 
+        path: rt_path,
+        uris: mock_uris,
+    });
+    grt2.setIndexes({ ...indexes, historyDB });
+    const connStream2 = await grt2.parse({ format: 'jsonld', objectMode: true });
+    let count2 = 0;
+    const endStream2 = new Promise(res => {
+        connStream2.on('end', () => res(true))
+            .on('error', () => res(false));
+    });
+    connStream2.on('data', conn => { count2++; });
+    const success2 = await endStream2;
+
+    expect(success1).toBeTruthy();
+    expect(count1).toBeGreaterThan(0);
+    expect(success2).toBeTruthy();
+    expect(count2).toBe(0);
+
+    await del(['./test/data/history.db'], { force: true });
+});
+
 test('Check all parsed connections are consistent regarding departure and arrival times', async () => {
     grt.setIndexes(memIndexes);
     let connStream = await grt.parse({ format: 'jsonld' });
@@ -641,8 +686,8 @@ test('Cover Gtfsrt2LC functions', async () => {
     }
     expect(fail).toBeDefined();
 
-    const readStream = new Readable({ objectMode: true, read() {}});
-    gtfsrt2lc.handleResponse({ 
+    const readStream = new Readable({ objectMode: true, read() { } });
+    gtfsrt2lc.handleResponse({
         statusCode: 200,
         headers: { 'content-encoding': 'fake-format' },
         body: Promise.resolve(readStream)