From 3027caf21857b01dad03438b8f70393a2cf5223e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Og=C3=B3rek?= Date: Thu, 19 Oct 2023 15:46:14 +0200 Subject: [PATCH] feat: Add support for jsonb data type to Airtable --- wrappers/dockerfiles/airtable/server.py | 24 ++++++- wrappers/src/fdw/airtable_fdw/README.md | 1 + wrappers/src/fdw/airtable_fdw/airtable_fdw.rs | 6 +- wrappers/src/fdw/airtable_fdw/result.rs | 15 ++++- wrappers/src/fdw/airtable_fdw/tests.rs | 63 +++++++++++++++---- 5 files changed, 90 insertions(+), 19 deletions(-) diff --git a/wrappers/dockerfiles/airtable/server.py b/wrappers/dockerfiles/airtable/server.py index bf66a479..9b2c700e 100644 --- a/wrappers/dockerfiles/airtable/server.py +++ b/wrappers/dockerfiles/airtable/server.py @@ -38,11 +38,29 @@ def do_GET(self): if __name__ == "__main__": # Populate a test table - client.create(test_table, {'field1': 1, 'field2': 'two', 'field3': '2023-07-19T06:39:15.000Z'}) - client.create(test_table, {'field1': 2, 'field2': 'three', 'field3': '2023-07-20T06:39:15.000Z'}) + client.create( + test_table, + { + "numeric_field": 1, + "string_field": "two", + "timestamp_field": "2023-07-19T06:39:15.000Z", + "strings_array_field": ["foo", "bar"], + "object_field": {"foo": "bar"}, + }, + ) + client.create( + test_table, + { + "numeric_field": 2, + "string_field": "three", + "timestamp_field": "2023-07-20T06:39:15.000Z", + "strings_array_field": ["baz", "qux"], + "object_field": {"foo": "baz"}, + }, + ) # Create a test view - airtablemock.create_view(base_id, test_table, test_view, 'field2 = "three"') + airtablemock.create_view(base_id, test_table, test_view, 'string_field = "three"') # Create web server webServer = HTTPServer((hostName, serverPort), AirtableMockServer) diff --git a/wrappers/src/fdw/airtable_fdw/README.md b/wrappers/src/fdw/airtable_fdw/README.md index c5eb441e..ac5dcc66 100644 --- a/wrappers/src/fdw/airtable_fdw/README.md +++ b/wrappers/src/fdw/airtable_fdw/README.md @@ -11,6 +11,7 @@ This is a foreign data wrapper for [Airtable](https://www.airtable.com). It is d | Version | Date | Notes | | ------- | ---------- | ---------------------------------------------------- | +| 0.1.3 | 2023-10-20 | Added jsonb data types support | | 0.1.2 | 2023-07-19 | Added more data types support | | 0.1.1 | 2023-07-13 | Added fdw stats collection | | 0.1.0 | 2022-11-30 | Initial version | diff --git a/wrappers/src/fdw/airtable_fdw/airtable_fdw.rs b/wrappers/src/fdw/airtable_fdw/airtable_fdw.rs index 2f679c00..7d485723 100644 --- a/wrappers/src/fdw/airtable_fdw/airtable_fdw.rs +++ b/wrappers/src/fdw/airtable_fdw/airtable_fdw.rs @@ -28,7 +28,7 @@ fn create_client(api_key: &str) -> Result for AirtableFdw { Some(api_key) => Some(create_client(api_key)?), None => { let key_id = require_option("api_key_id", options)?; - if let Some(api_key) = get_vault_secret(&key_id) { + if let Some(api_key) = get_vault_secret(key_id) { Some(create_client(&api_key)?) } else { None @@ -129,7 +129,7 @@ impl ForeignDataWrapper for AirtableFdw { let base_id = require_option("base_id", options)?; let table_id = require_option("table_id", options)?; let view_id = options.get("view_id"); - let url = self.build_url(&base_id, &table_id, view_id); + let url = self.build_url(base_id, table_id, view_id); let mut rows = Vec::new(); if let Some(client) = &self.client { diff --git a/wrappers/src/fdw/airtable_fdw/result.rs b/wrappers/src/fdw/airtable_fdw/result.rs index 7197285f..4d76f38b 100644 --- a/wrappers/src/fdw/airtable_fdw/result.rs +++ b/wrappers/src/fdw/airtable_fdw/result.rs @@ -84,6 +84,7 @@ impl<'de> Deserialize<'de> for AirtableFields { } } +// Available Airtable field types: https://airtable.com/developers/web/api/field-model impl AirtableRecord { pub(super) fn to_row(&self, columns: &[Column]) -> AirtableFdwResult { let mut row = Row::new(); @@ -212,7 +213,19 @@ impl AirtableRecord { } }, ), - _ => return Err(AirtableFdwError::UnsupportedColumnType(col.name.clone())), + pg_sys::JSONBOID => self.fields.0.get(&col.name).map_or_else( + || Ok(None), + |val| { + if val.is_array() || val.is_object() { + Ok(Some(Cell::Json(pgrx::JsonB(val.clone())))) + } else { + Err(()) + } + }, + ), + _ => { + return Err(AirtableFdwError::UnsupportedColumnType(col.name.clone())); + } } .map_err(|_| AirtableFdwError::ColumnTypeNotMatch(col.name.clone()))?; diff --git a/wrappers/src/fdw/airtable_fdw/tests.rs b/wrappers/src/fdw/airtable_fdw/tests.rs index 8013473f..7095d7d5 100644 --- a/wrappers/src/fdw/airtable_fdw/tests.rs +++ b/wrappers/src/fdw/airtable_fdw/tests.rs @@ -1,8 +1,7 @@ #[cfg(any(test, feature = "pg_test"))] #[pgrx::pg_schema] mod tests { - use pgrx::pg_test; - use pgrx::prelude::*; + use pgrx::{prelude::*, JsonB}; #[pg_test] fn airtable_smoketest() { @@ -28,9 +27,11 @@ mod tests { c.update( r#" CREATE FOREIGN TABLE airtable_table ( - field1 numeric, - field2 text, - field3 timestamp + numeric_field numeric, + string_field text, + timestamp_field timestamp, + strings_array_field jsonb, + object_field jsonb ) SERVER airtable_server OPTIONS ( @@ -45,9 +46,11 @@ mod tests { c.update( r#" CREATE FOREIGN TABLE airtable_view ( - field1 numeric, - field2 text, - field3 timestamp + numeric_field numeric, + string_field text, + timestamp_field timestamp, + strings_array_field jsonb, + object_field jsonb ) SERVER airtable_server OPTIONS ( @@ -66,19 +69,55 @@ mod tests { */ let results = c .select( - "SELECT field2 FROM airtable_table WHERE field1 = 1", + "SELECT string_field FROM airtable_table WHERE numeric_field = 1", None, None, ) .unwrap() - .filter_map(|r| r.get_by_name::<&str, _>("field2").unwrap()) + .filter_map(|r| r.get_by_name::<&str, _>("string_field").unwrap()) .collect::>(); assert_eq!(results, vec!["two"]); let results = c - .select("SELECT field2 FROM airtable_view", None, None) + .select( + "SELECT strings_array_field FROM airtable_table WHERE numeric_field = 1", + None, + None, + ) + .unwrap() + .filter_map(|r| { + r.get_by_name::("strings_array_field") + .expect("strings_array_field is missing") + .and_then(|v| serde_json::from_value::>(v.0.to_owned()).ok()) + }) + .collect::>(); + + assert_eq!(results, vec![vec!["foo", "bar"]]); + + #[derive(serde::Deserialize)] + struct Foo { + foo: String, + } + + let results = c + .select( + "SELECT object_field FROM airtable_table WHERE numeric_field = 1", + None, + None, + ) + .unwrap() + .filter_map(|r| { + r.get_by_name::("object_field") + .expect("object_field is missing") + .and_then(|v| serde_json::from_value::(v.0.to_owned()).ok()) + }) + .collect::>(); + assert_eq!(results[0].foo, "bar"); + + let results = c + .select("SELECT string_field FROM airtable_view", None, None) .unwrap() - .filter_map(|r| r.get_by_name::<&str, _>("field2").unwrap()) + .filter_map(|r| r.get_by_name::<&str, _>("string_field").unwrap()) .collect::>(); assert_eq!(results, vec!["three"]); });