feat(gatsby-transformer-csv): add tsv parsing option (#21198)

* feat(gatsby-transformer-csv): add tsv parsing option Allow TSV files to be transformed using the gatsby-transformer-csv plugin. The existing code filters out any non-csv files, so passing the delimiter directly would not work. * improvement(gatsby-transformer-csv): allow filtering by extension so that tsv files can be parsed
gatsbyjs · Feb 12, 2020 · fef77ba · fef77ba
1 parent 8bbccc8
commit fef77ba
Show file tree

Hide file tree

Showing 5 changed files with 380 additions and 11 deletions.
diff --git a/packages/gatsby-transformer-csv/README.md b/packages/gatsby-transformer-csv/README.md
@@ -28,9 +28,14 @@ module.exports = {
 }
 ```
 
-Above is the minimal configuration required to begin working. Additional
+### Configuration
+
+The example above is the minimal configuration required to begin working. Additional
 customization of the parsing process is possible using the parameters listed in
-[csvtojson](https://github.com/Keyang/node-csvtojson#parameters).
+[csvtojson](https://github.com/Keyang/node-csvtojson#parameters). Any parameter listed on that page
+can be passed directly to the library using the plugin options.
+
+For example, to pass the `noheader` option, you can configure like so:
 
 ```javascript
 // In your gatsby-config.js
@@ -53,6 +58,32 @@ module.exports = {
 }
 ```
 
+By default, files that do not have a `.csv` extension will not be parsed, but this can be configured using the `extensions` option which takes an array of strings.
+
+For example, if you need to parse TSV files, you can configure the plugin like so:
+
+```javascript
+// In your gatsby-config.js
+module.exports = {
+  plugins: [
+    {
+      resolve: `gatsby-source-filesystem`,
+      options: {
+        name: `data`,
+        path: `${__dirname}/src/data/`,
+      },
+    },
+    {
+      resolve: `gatsby-transformer-csv`,
+      options: {
+        extensions: [`tsv`],
+        delimiter: '\t'
+      },
+    },
+  ];
+}
+```
+
 You can see an example project at https://github.com/gatsbyjs/gatsby/tree/master/examples/using-csv.
 
 ## Parsing algorithm

diff --git a/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap b/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap
@@ -1,5 +1,141 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`Process nodes correctly allows other extensions to be used for input files 1`] = `
+Array [
+  Array [
+    Object {
+      "children": Array [],
+      "field1": "blue",
+      "field2": "funny",
+      "id": "uuid-from-gatsby",
+      "internal": Object {
+        "contentDigest": "contentDigest",
+        "type": "TestKsv",
+      },
+      "parent": "whatever",
+    },
+  ],
+  Array [
+    Object {
+      "children": Array [],
+      "field1": "true",
+      "field2": "yup",
+      "id": "uuid-from-gatsby",
+      "internal": Object {
+        "contentDigest": "contentDigest",
+        "type": "TestKsv",
+      },
+      "parent": "whatever",
+    },
+  ],
+  Array [
+    Object {
+      "children": Array [],
+      "field1": "false",
+      "field2": "nope",
+      "id": "uuid-from-gatsby",
+      "internal": Object {
+        "contentDigest": "contentDigest",
+        "type": "TestKsv",
+      },
+      "parent": "whatever",
+    },
+  ],
+]
+`;
+
+exports[`Process nodes correctly allows other extensions to be used for input files 2`] = `
+Array [
+  Array [
+    Object {
+      "child": Object {
+        "children": Array [],
+        "field1": "blue",
+        "field2": "funny",
+        "id": "uuid-from-gatsby",
+        "internal": Object {
+          "contentDigest": "contentDigest",
+          "type": "TestKsv",
+        },
+        "parent": "whatever",
+      },
+      "parent": Object {
+        "children": Array [],
+        "content": "blue	funny
+true	yup
+false	nope",
+        "extension": "ksv",
+        "id": "whatever",
+        "internal": Object {
+          "contentDigest": "whatever",
+          "mediaType": "text/ksv",
+        },
+        "name": "test",
+        "parent": null,
+      },
+    },
+  ],
+  Array [
+    Object {
+      "child": Object {
+        "children": Array [],
+        "field1": "true",
+        "field2": "yup",
+        "id": "uuid-from-gatsby",
+        "internal": Object {
+          "contentDigest": "contentDigest",
+          "type": "TestKsv",
+        },
+        "parent": "whatever",
+      },
+      "parent": Object {
+        "children": Array [],
+        "content": "blue	funny
+true	yup
+false	nope",
+        "extension": "ksv",
+        "id": "whatever",
+        "internal": Object {
+          "contentDigest": "whatever",
+          "mediaType": "text/ksv",
+        },
+        "name": "test",
+        "parent": null,
+      },
+    },
+  ],
+  Array [
+    Object {
+      "child": Object {
+        "children": Array [],
+        "field1": "false",
+        "field2": "nope",
+        "id": "uuid-from-gatsby",
+        "internal": Object {
+          "contentDigest": "contentDigest",
+          "type": "TestKsv",
+        },
+        "parent": "whatever",
+      },
+      "parent": Object {
+        "children": Array [],
+        "content": "blue	funny
+true	yup
+false	nope",
+        "extension": "ksv",
+        "id": "whatever",
+        "internal": Object {
+          "contentDigest": "whatever",
+          "mediaType": "text/ksv",
+        },
+        "name": "test",
+        "parent": null,
+      },
+    },
+  ],
+]
+`;
+
 exports[`Process nodes correctly correctly creates nodes from JSON which is an array of objects 1`] = `
 Array [
   Array [
@@ -1106,3 +1242,139 @@ c,23",
   ],
 ]
 `;
+
+exports[`Process nodes correctly passes through the delimiter option 1`] = `
+Array [
+  Array [
+    Object {
+      "children": Array [],
+      "field1": "blue",
+      "field2": "funny",
+      "id": "uuid-from-gatsby",
+      "internal": Object {
+        "contentDigest": "contentDigest",
+        "type": "TestTsv",
+      },
+      "parent": "whatever",
+    },
+  ],
+  Array [
+    Object {
+      "children": Array [],
+      "field1": "true",
+      "field2": "yup",
+      "id": "uuid-from-gatsby",
+      "internal": Object {
+        "contentDigest": "contentDigest",
+        "type": "TestTsv",
+      },
+      "parent": "whatever",
+    },
+  ],
+  Array [
+    Object {
+      "children": Array [],
+      "field1": "false",
+      "field2": "nope",
+      "id": "uuid-from-gatsby",
+      "internal": Object {
+        "contentDigest": "contentDigest",
+        "type": "TestTsv",
+      },
+      "parent": "whatever",
+    },
+  ],
+]
+`;
+
+exports[`Process nodes correctly passes through the delimiter option 2`] = `
+Array [
+  Array [
+    Object {
+      "child": Object {
+        "children": Array [],
+        "field1": "blue",
+        "field2": "funny",
+        "id": "uuid-from-gatsby",
+        "internal": Object {
+          "contentDigest": "contentDigest",
+          "type": "TestTsv",
+        },
+        "parent": "whatever",
+      },
+      "parent": Object {
+        "children": Array [],
+        "content": "blue	funny
+true	yup
+false	nope",
+        "extension": "tsv",
+        "id": "whatever",
+        "internal": Object {
+          "contentDigest": "whatever",
+          "mediaType": "text/tsv",
+        },
+        "name": "test",
+        "parent": null,
+      },
+    },
+  ],
+  Array [
+    Object {
+      "child": Object {
+        "children": Array [],
+        "field1": "true",
+        "field2": "yup",
+        "id": "uuid-from-gatsby",
+        "internal": Object {
+          "contentDigest": "contentDigest",
+          "type": "TestTsv",
+        },
+        "parent": "whatever",
+      },
+      "parent": Object {
+        "children": Array [],
+        "content": "blue	funny
+true	yup
+false	nope",
+        "extension": "tsv",
+        "id": "whatever",
+        "internal": Object {
+          "contentDigest": "whatever",
+          "mediaType": "text/tsv",
+        },
+        "name": "test",
+        "parent": null,
+      },
+    },
+  ],
+  Array [
+    Object {
+      "child": Object {
+        "children": Array [],
+        "field1": "false",
+        "field2": "nope",
+        "id": "uuid-from-gatsby",
+        "internal": Object {
+          "contentDigest": "contentDigest",
+          "type": "TestTsv",
+        },
+        "parent": "whatever",
+      },
+      "parent": Object {
+        "children": Array [],
+        "content": "blue	funny
+true	yup
+false	nope",
+        "extension": "tsv",
+        "id": "whatever",
+        "internal": Object {
+          "contentDigest": "whatever",
+          "mediaType": "text/tsv",
+        },
+        "name": "test",
+        "parent": null,
+      },
+    },
+  ],
+]
+`;