diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index a2e0802..db75268 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [12.x, 14.x, 16.x] + node-version: [16.x] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: - name: Checkout diff --git a/dependabot.yml b/dependabot.yml new file mode 100644 index 0000000..3995f9e --- /dev/null +++ b/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: "npm" + directory: "/" + schedule: + # Check for updates once a week + interval: "weekly" + # Raise pull requests for version updates against the `development` branch + target-branch: "development" + # Labels on pull requests for security and version updates + labels: + - "npm dependencies" \ No newline at end of file diff --git a/lib/GtfsIndex.js b/lib/GtfsIndex.js index d704ee7..547ed69 100644 --- a/lib/GtfsIndex.js +++ b/lib/GtfsIndex.js @@ -73,7 +73,7 @@ class GtfsIndex { let stop_times_index = null; let tripsByRoute = null; let firstStops = null; - let calendar = null; + let calendar_index = null; let calendarDates = null; let tp = null; let stp = null; @@ -84,7 +84,6 @@ class GtfsIndex { if (deduce) { tripsByRoute = new Map(); firstStops = new Map(); - calendar = new Map(); calendarDates = new Map(); } @@ -96,6 +95,8 @@ class GtfsIndex { routes_index = new Map(); trips_index = new Map(); stop_times_index = new Map(); + calendar_index = new Map(); + if (uTrips) { console.error(`Using grep to extract the stop_times of ${uTrips.length} trips`); @@ -126,6 +127,7 @@ class GtfsIndex { routes_index = new Level('.rt_indexes/.routes', { valueEncoding: 'json' }); trips_index = new Level('.rt_indexes/.trips', { valueEncoding: 'json' }); stop_times_index = new Level('.rt_indexes/.stop_times', { valueEncoding: 'json' }); + calendar_index = new Level('.rt_indexes/.calendar', { valueEncoding: 'json' }); tp = this.createIndex(this.auxPath + '/trips.txt', trips_index, 'trip_id', tripsByRoute); // Make sure stop_times.txt is ordered by stop_sequence @@ -139,9 +141,9 @@ class GtfsIndex { let sp = this.createIndex(this.auxPath + '/stops.txt', stops_index, 'stop_id'); let rp = this.createIndex(this.auxPath + '/routes.txt', routes_index, 'route_id'); + cp = this.createIndex(this.auxPath + '/calendar.txt', calendar_index, 'service_id'); - if (deduce) { - cp = this.createIndex(this.auxPath + '/calendar.txt', calendar, 'service_id'); + if (deduce) { cdp = this.processCalendarDates(this.auxPath + '/calendar_dates.txt', calendarDates); } @@ -153,9 +155,9 @@ class GtfsIndex { "trips": trips_index, "stops": stops_index, "stop_times": stop_times_index, + "calendar": calendar_index, "tripsByRoute": tripsByRoute, "firstStops": firstStops, - "calendar": calendar, "calendarDates": calendarDates }; } diff --git a/lib/Gtfsrt2LC.js b/lib/Gtfsrt2LC.js index 2371f12..8e66293 100644 --- a/lib/Gtfsrt2LC.js +++ b/lib/Gtfsrt2LC.js @@ -7,7 +7,7 @@ const zlib = require('zlib'); const gtfsrt = require('gtfs-realtime-bindings').transit_realtime; const JSONStream = require('JSONStream'); const N3 = require('n3'); -const { format, addHours, addMinutes, addSeconds } = require('date-fns'); +const { format, addHours, addMinutes, addSeconds, addDays } = require('date-fns'); const Connections2JSONLD = require('./Connections2JSONLD'); const Connections2CSV = require('./Connections2CSV'); const Connections2Triples = require('./Connections2Triples'); @@ -40,7 +40,7 @@ class Gtfsrt2LC { "routeLabel": "routes.route_long_name.replace(/\\s/gi, '');", "tripLabel": "routes.route_short_name + routes.route_id;", "tripStartTime": "format(trips.startTime, \"yyyyMMdd'T'HHmm\");" - } + } }; } @@ -79,8 +79,7 @@ class Gtfsrt2LC { if (entity.tripUpdate) { let tripUpdate = entity.tripUpdate; let tripId = tripUpdate.trip.tripId; - let startDate = tripUpdate.trip.startDate; - let serviceDay = new Date(startDate.substr(0, 4), parseInt(startDate.substr(4, 2)) - 1, startDate.substr(6, 2)); + const timestamp = tripUpdate.timestamp || this.jsonData.header.timestamp; // Check if tripId is directly provided or has to be found from GTFS source, @@ -89,11 +88,6 @@ class Gtfsrt2LC { let deduced = await this.deduceTripId(tripUpdate.trip.routeId, tripUpdate.trip.startTime, tripUpdate.trip.startDate, tripUpdate.trip.directionId); tripId = deduced ? deduced['trip_id'] : null; - // Correct startTime by adding 24 hours (error noticed for HSL Helsinki GTFS-RT) - if (deduced && deduced['startTime']) { - tripUpdate.trip.startTime = deduced['startTime']; - this.correctTimes(tripUpdate); - } } let r = null; @@ -115,11 +109,35 @@ class Gtfsrt2LC { return; } + // Figure service date and trip start time + let serviceDay = null; let tripStartTime = null; - if (tripUpdate.trip.startTime) { - tripStartTime = this.addDuration(serviceDay, this.parseGTFSDuration(tripUpdate.trip.startTime)); + + if (tripUpdate.trip.startDate) { + const rawStartDate = tripUpdate.trip.startDate; + serviceDay = new Date( + rawStartDate.substr(0, 4), + parseInt(rawStartDate.substr(4, 2)) - 1, + rawStartDate.substr(6, 2) + ); + if (tripUpdate.trip.startTime) { + tripStartTime = this.addDuration(serviceDay, this.parseGTFSDuration(tripUpdate.trip.startTime)); + } else { + // Extract from static data + tripStartTime = this.addDuration( + serviceDay, + this.parseGTFSDuration((await this.getStopTimes(tripId))[0]['departure_time']) + ); + } } else { - tripStartTime = this.addDuration(serviceDay, this.parseGTFSDuration((await this.getStopTimes(tripId))[0]['departure_time'])); + // Extract from static data + const serviceObj = this.calendar.get(t['service_id']); + const depTime = (await this.getStopTimes(tripId))[0]['departure_time']; + serviceDay = this.findTripStartDate(depTime, serviceObj); + tripStartTime = this.addDuration( + serviceDay, + this.parseGTFSDuration((await this.getStopTimes(tripId))[0]['departure_time']) + ); } // Check if the trip has been canceled or not @@ -179,7 +197,7 @@ class Gtfsrt2LC { // Add start time to trip object t.startTime = tripStartTime; - + // Build JSON Connection let json_connection = { type, @@ -342,7 +360,7 @@ class Gtfsrt2LC { // Find the trip that runs on the same day let today = new Date(startDate.substring(0, 4), parseInt(startDate.substring(4, 6)) - 1, startDate.substring(6, 8)); for (let i = 0; i < trips.length; i++) { - const service = this.calendar.get(trips[i]['service_id']); + const service = await this.calendar.get(trips[i]['service_id']); const exceptions = this.calendarDates.get(trips[i]['service_id']) || {}; const minDate = new Date(service['start_date'].substring(0, 4), parseInt(service['start_date'].substring(4, 6)) - 1, service['start_date'].substring(6, 8)); const maxDate = new Date(service['end_date'].substring(0, 4), parseInt(service['end_date'].substring(4, 6)) - 1, service['end_date'].substring(6, 8)); @@ -370,12 +388,34 @@ class Gtfsrt2LC { } } - correctTimes(tripUpdate) { - for (let i = 0; i < tripUpdate.stopTimeUpdate.length; i++) { - let dep = parseInt(tripUpdate.stopTimeUpdate[i].departure.time) + 86400; - let arr = parseInt(tripUpdate.stopTimeUpdate[i].arrival.time) + 86400; - tripUpdate.stopTimeUpdate[i].departure.time = dep; - tripUpdate.stopTimeUpdate[i].arrival.time = arr; + findTripStartDate(depTime, service) { + const now = new Date(); + const days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']; + const today = format(now, 'EEEE').toLowerCase(); + const tomorrow = days[days.indexOf(today) > 5 ? 0 : days.indexOf(today) + 1]; + const yesterday = days[days.indexOf(today) < 1 ? 6 : days.indexOf(today) - 1]; + + const todayServiceDate = this.addDuration( + new Date(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()), + this.parseGTFSDuration(depTime) + ); + const tomorrowServiceDate = addDays(todayServiceDate, 1); + const yesterdayServiceDate = addDays(todayServiceDate, -1); + + const todayDistance = service[today] === '1' ? Math.abs(now - todayServiceDate) : Number.POSITIVE_INFINITY; + const tomorrowDistance = service[tomorrow] === '1' ? Math.abs(now - tomorrowServiceDate) : Number.POSITIVE_INFINITY; + const yesterdayDistance = service[yesterday] === '1' ? Math.abs(now - yesterdayServiceDate) : Number.POSITIVE_INFINITY; + + if(todayDistance === Math.min(todayDistance, tomorrowDistance, yesterdayDistance)) { + return todayServiceDate.setUTCHours(0, 0, 0 ,0); + } + + if(tomorrowDistance === Math.min(todayDistance, tomorrowDistance, yesterdayDistance)) { + return tomorrowServiceDate.setUTCHours(0, 0, 0 ,0); + } + + if(yesterdayDistance === Math.min(todayDistance, tomorrowDistance, yesterdayDistance)) { + return yesterdayServiceDate.setUTCHours(0, 0, 0 ,0); } } @@ -785,6 +825,10 @@ class Gtfsrt2LC { return this._stops; } + set stops(stops) { + this._stops = stops; + } + get stop_times() { return this._stop_times; } diff --git a/package-lock.json b/package-lock.json index af76f84..541f6ff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "gtfsrt2lc", - "version": "2.0.5", + "version": "2.1.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "gtfsrt2lc", - "version": "2.0.5", + "version": "2.1.1", "license": "MIT", "dependencies": { "commander": "^5.0.0", @@ -1644,6 +1644,17 @@ "node": ">=0.2.0" } }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -3696,9 +3707,9 @@ "dev": true }, "node_modules/json5": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", - "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true, "bin": { "json5": "lib/cli.js" @@ -4780,6 +4791,14 @@ "node": ">=10" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -5081,9 +5100,12 @@ } }, "node_modules/undici": { - "version": "5.8.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.8.2.tgz", - "integrity": "sha512-3KLq3pXMS0Y4IELV045fTxqz04Nk9Ms7yfBBHum3yxsTR4XNn+ZCaUbf/mWitgYDAhsplQ0B1G4S5D345lMO3A==", + "version": "5.19.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.19.1.tgz", + "integrity": "sha512-YiZ61LPIgY73E7syxCDxxa3LV2yl3sN8spnIuTct60boiiRaE1J8mNWHO8Im2Zi/sFrPusjLlmRPrsyraSqX6A==", + "dependencies": { + "busboy": "^1.6.0" + }, "engines": { "node": ">=12.18" } @@ -6709,6 +6731,14 @@ "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==" }, + "busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "requires": { + "streamsearch": "^1.1.0" + } + }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -8264,9 +8294,9 @@ "dev": true }, "json5": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", - "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true }, "jsonparse": { @@ -9063,6 +9093,11 @@ "escape-string-regexp": "^2.0.0" } }, + "streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==" + }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -9280,9 +9315,12 @@ } }, "undici": { - "version": "5.8.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.8.2.tgz", - "integrity": "sha512-3KLq3pXMS0Y4IELV045fTxqz04Nk9Ms7yfBBHum3yxsTR4XNn+ZCaUbf/mWitgYDAhsplQ0B1G4S5D345lMO3A==" + "version": "5.19.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.19.1.tgz", + "integrity": "sha512-YiZ61LPIgY73E7syxCDxxa3LV2yl3sN8spnIuTct60boiiRaE1J8mNWHO8Im2Zi/sFrPusjLlmRPrsyraSqX6A==", + "requires": { + "busboy": "^1.6.0" + } }, "universalify": { "version": "0.1.2", diff --git a/package.json b/package.json index 24c1717..a0c5893 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gtfsrt2lc", - "version": "2.0.5", + "version": "2.1.1", "description": "Converts the GTFS-RT to Linked Connections", "main": "./Gtfsrt2LC.js", "bin": { diff --git a/test/data/bustang.gtfs.zip b/test/data/bustang.gtfs.zip new file mode 100644 index 0000000..a7aa0d7 Binary files /dev/null and b/test/data/bustang.gtfs.zip differ diff --git a/test/data/bustang.pb b/test/data/bustang.pb new file mode 100644 index 0000000..d375a85 Binary files /dev/null and b/test/data/bustang.pb differ diff --git a/test/gtfsrt2lc.test.js b/test/gtfsrt2lc.test.js index c2c0a2e..420b57a 100644 --- a/test/gtfsrt2lc.test.js +++ b/test/gtfsrt2lc.test.js @@ -1,5 +1,6 @@ const fs = require('fs'); const del = require('del'); +const { Readable } = require('stream'); const uri_templates = require('uri-templates'); const GtfsIndex = require('../lib/GtfsIndex'); const Gtfsrt2lc = require('../lib/Gtfsrt2LC'); @@ -216,6 +217,7 @@ test('Check all parsed connections are consistent regarding departure and arriva await levelIndexes.trips.close(); await levelIndexes.stops.close(); await levelIndexes.stop_times.close(); + await levelIndexes.calendar.close(); }); test('Parse real-time update (test/data/realtime_rawdata) and give it back in jsonld format (no objectMode)', async () => { @@ -469,6 +471,35 @@ test('Check cancelled vehicle detection and related Connections (use test/data/c expect(cancelledConnections.length).toBe(9); }); +test('Test processing of feed without trip start date and time (use test/data/bustang.pb) with MemStore', async () => { + grt = new Gtfsrt2lc({ path: './test/data/bustang.pb', uris: mock_uris }); + let gti = new GtfsIndex({ path: './test/data/bustang.gtfs.zip' }); + const indexes = await gti.getIndexes({ store: 'MemStore' }); + grt.setIndexes(indexes); + + let connStream = await grt.parse({ format: 'turtle', objectMode: true }); + let connections = []; + + expect.assertions(2); + + connStream.on('data', conn => { + connections.push(conn); + }); + + let stream_end = new Promise(resolve => { + connStream.on('end', () => { + resolve(true); + }); + connStream.on('error', () => { + resolve(false); + }); + }); + + let finished = await stream_end; + expect(finished).toBeTruthy(); + expect(connections.length).toBe(365); +}); + test('Test parsing a GTFS-RT v2.0 file (use test/data/realtime_rawdata_v2) with MemStore', async () => { grt = new Gtfsrt2lc({ path: './test/data/realtime_rawdata_v2', uris: mock_uris }); let gti = new GtfsIndex({ path: './test/data/static_rawdata_v2.zip' }); @@ -564,6 +595,63 @@ test('Test measures to produce consistent connections', () => { expect(update['arrival']['time']).toBe(new Date('2020-03-03T08:21:00.000Z').getTime() / 1000); }); +test('Non-existent gtfs-rt file throws exception', async () => { + grt = new Gtfsrt2lc({ path: './data/path/to/fake.file', uris: mock_uris }); + let gti = new GtfsIndex({ path: './test/data/bustang.gtfs.zip' }); + const indexes = await gti.getIndexes({ store: 'MemStore' }); + grt.setIndexes(indexes); + + let failed = null; + + try { + const connStream = await grt.parse({ format: 'json', objectMode: true }); + } catch (err) { + failed = err; + } + + expect(failed).toBeDefined() +}); + +test('Missing index throws exception', async () => { + grt = new Gtfsrt2lc({ path: './data/path/bustang.pb', uris: mock_uris }); + let gti = new GtfsIndex({ path: './test/data/bustang.gtfs.zip' }); + const indexes = await gti.getIndexes({ store: 'MemStore' }); + grt.setIndexes(indexes); + grt.stops = null; + + let failed = null; + + try { + const connStream = await grt.parse({ format: 'json', objectMode: true }); + } catch (err) { + failed = err; + } + + expect(failed).toBeDefined() +}); + +test('Cover Gtfsrt2LC functions', async () => { + const gtfsrt2lc = new Gtfsrt2lc({}); + let fail = null; + + try { + await gtfsrt2lc.handleResponse({ statusCode: 401 }); + } catch (err) { + fail = err; + } + expect(fail).toBeDefined(); + + const readStream = new Readable({ objectMode: true, read() {}}); + gtfsrt2lc.handleResponse({ + statusCode: 200, + headers: { 'content-encoding': 'fake-format' }, + body: Promise.resolve(readStream) + }).then(result => { + expect(result).toBe(false); + }); + readStream.push(null); +}); + test('Cover GtfsIndex functions', async () => { let gti = new GtfsIndex({ path: 'https://gtfs.irail.be/nmbs/gtfs/latest.zip' }); try {