Merge pull request #77 from linkedconnections/development

v2.1.5

.github/workflows/build-test.yml

@@ -14,34 +14,17 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [16.x, 18.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node-version }}
           cache: 'npm'
       - run: npm ci
       - run: npm run build --if-present
       - run: npm run test-ci
-
-      - name: Submit coverage results
-        uses: coverallsapp/github-action@master
-        with:
-          github-token: ${{ secrets.github_token }}
-          flag-name: run-${{ matrix.node-version }}
-          parallel: true
-
-  coveralls:
-    needs: test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Consolidate test coverage from different jobs
-        uses: coverallsapp/github-action@master
-        with:
-          github-token: ${{ secrets.github_token }}
-          parallel-finished: true

README.md

@@ -1,6 +1,6 @@
 # gtfsrt2lc
 
-[![Node.js CI](https://github.com/linkedconnections/gtfsrt2lc/actions/workflows/build-test.yml/badge.svg)](https://github.com/linkedconnections/gtfsrt2lc/actions/workflows/build-test.yml) [![npm](https://img.shields.io/npm/v/gtfsrt2lc.svg?style=popout)](https://npmjs.com/package/gtfsrt2lc) [![Coverage Status](https://coveralls.io/repos/github/linkedconnections/gtfsrt2lc/badge.svg?branch=master)](https://coveralls.io/github/linkedconnections/gtfsrt2lc?branch=master)
+[![Node.js CI](https://github.com/linkedconnections/gtfsrt2lc/actions/workflows/build-test.yml/badge.svg)](https://github.com/linkedconnections/gtfsrt2lc/actions/workflows/build-test.yml) [![npm](https://img.shields.io/npm/v/gtfsrt2lc.svg?style=popout)](https://npmjs.com/package/gtfsrt2lc)
 
 Converts [GTFS-RT](https://developers.google.com/transit/gtfs-realtime/) updates to [Linked Connections](http://linkedconnections.org/) following a predefined URI strategy that is provided using the [RFC 6570](https://tools.ietf.org/html/rfc6570) specification and any variable present in a (also given) related [GTFS](https://developers.google.com/tansit/gtfs/reference/) data source.
 

jest.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+    setupFiles: ['./jest.polyfills.js'],
+}

jest.polyfills.js

@@ -0,0 +1,30 @@
+/**
+ * @note The block below contains polyfills for Node.js globals
+ * required for Jest to function when running JSDOM tests.
+ * These HAVE to be require's and HAVE to be in this exact
+ * order, since "undici" depends on the "TextEncoder" global API.
+ *
+ * Consider migrating to a more modern test runner if
+ * you don't want to deal with this.
+ */
+
+const { TextDecoder, TextEncoder, ReadableStream } = require("node:util")
+
+Object.defineProperties(globalThis, {
+    TextDecoder: { value: TextDecoder },
+    TextEncoder: { value: TextEncoder },
+    ReadableStream: { value: ReadableStream },
+})
+
+const { Blob, File } = require("node:buffer")
+const { fetch, Headers, FormData, Request, Response } = require("undici")
+
+Object.defineProperties(globalThis, {
+    fetch: { value: fetch, writable: true },
+    Blob: { value: Blob },
+    File: { value: File },
+    Headers: { value: Headers },
+    FormData: { value: FormData },
+    Request: { value: Request },
+    Response: { value: Response },
+});

lib/GtfsIndex.js

@@ -33,7 +33,7 @@ class GtfsIndex {
                     throw new Error('Please provide a valid url or a path to a GTFS feed');
                 } else {
                     if (this.path.endsWith('.zip')) {
-                        await this.unzip(fs.createReadStream(this.path));
+                        Utils.unzip(this.path, this.auxPath);
                     } else {
                         this.auxPath = this.path;
                     }
@@ -47,23 +47,29 @@ class GtfsIndex {
         }
     }
 
-    async download(url, headers) {
-        try {
-            const res = await request(url, { method: 'GET', headers });
-
-            if (res.statusCode === 200) {
-                await this.unzip(await res.body);
-            } else {
-                throw new Error(`Error on HTTP request: ${url}, Message: ${await res.body.text()}`);
+    download(url, headers) {
+        return new Promise(async (resolve, reject) => {
+            try {
+                const res = await request(url, {
+                    method: 'GET',
+                    headers,
+                    maxRedirections: 10
+                });
+
+                if (res.statusCode === 200) {
+                    res.body.pipe(createWriteStream("/tmp/gtfs.zip"))
+                        .on("finish", () => {
+                            Utils.unzip("/tmp/gtfs.zip", this.auxPath);
+                            resolve();
+                        });
+                } else {
+                    reject(new Error(`Error on HTTP request: ${url}, Message: ${await res.body.text()}`));
+                }
+            } catch (err) {
+                await this.cleanUp();
+                reject(err);
             }
-        } catch (err) {
-            await this.cleanUp();
-            throw err;
-        }
-    }
-
-    unzip(stream) {
-        return Utils.unzipStream(stream, this.auxPath);
+        });
     }
 
     async createIndexes(store, uTrips, deduce) {
@@ -143,7 +149,7 @@ class GtfsIndex {
         let rp = this.createIndex(this.auxPath + '/routes.txt', routes_index, 'route_id');
         cp = this.createIndex(this.auxPath + '/calendar.txt', calendar_index, 'service_id');
 
-        if (deduce) {    
+        if (deduce) {
             cdp = this.processCalendarDates(this.auxPath + '/calendar_dates.txt', calendarDates);
         }
 
@@ -303,7 +309,7 @@ class GtfsIndex {
     async cleanUp() {
         // We don't want to delete sources that are probably been reused
         if (this.auxPath !== this.path) {
-            await del([this.auxPath], { force: true });
+            await del([this.auxPath, "/tmp/gtfs.zip"], { force: true });
         }
     }
 

lib/Gtfsrt2LC.js

@@ -273,7 +273,11 @@ class Gtfsrt2LC {
          if (!download_url) {
             reject('Please provide a valid url or a path to a GTFS-RT feed');
          } else {
-            const res = await request(this.path, { method: 'GET', headers: this.headers });
+            const res = await request(this.path, { 
+               method: 'GET', 
+               headers: this.headers,
+               maxRedirections: 10
+            });
             return await this.handleResponse(res);
          }
       } else {

lib/Utils.js

@@ -1,21 +1,11 @@
 const fs = require('fs');
-const unzip = require('unzipper');
 const { format } = require('date-fns');
+const AdmZip = require('adm-zip');
 
-function unzipStream(stream, outPath) {
-    return new Promise((resolve, reject) => {
-        if (!fs.existsSync(outPath)) {
-            fs.mkdirSync(outPath);
-        }
-        stream.pipe(unzip.Extract({ path: outPath }))
-            .on('error', async err => {
-                reject(err);
-            })
-            .on('close', () => {
-                resolve();
-            });
-    });
-}
+function unzip(zipped, path) {
+    const adm = new AdmZip(zipped);
+    adm.extractAllTo(path, true);
+ }
 
 function resolveURI(template, raw, resolve, stopType) {
     let varNames = template.varNames;
@@ -90,7 +80,7 @@ function resolveScheduleRelationship(value) {
 }
 
 module.exports = {
-    unzipStream,
+    unzip,
     resolveURI,
     resolveScheduleRelationship
 }