Merge pull request #93 from trocco-io/add_column_type_numeric

Add NUMERIC Type

Author: HokutoMorita

.github/workflows/gem-push.yml

@@ -0,0 +1,34 @@
+name: Ruby Gem
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - '*'
+  pull_request:
+    branches:
+      - '*'
+    types: [ opened, synchronize ]
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    if: ${{ github.event_name == 'workflow_dispatch' || contains(github.ref, 'tags/v') }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+      - name: push gem
+        uses: trocco-io/push-gem-to-gpr-action@v1
+        with:
+          language: java
+          gem-path: "./build/gems/*.gem"
+          github-token: "${{ secrets.GITHUB_TOKEN }}"

README.md

@@ -116,18 +116,20 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
 
 - **column_options**: advanced: an array of options for columns
   - **name**: column name
-  - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, and `RECORD`. See belows for supported conversion type.
+  - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, `RECORD`, and `NUMERIC`. See belows for supported conversion type.
     - boolean (x):   `BOOLEAN`, `STRING` (default: `BOOLEAN`)
     - long (x):      `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `INTEGER`)
     - double (x):    `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `FLOAT`)
     - string:    `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, `RECORD` (default: `STRING`)
     - timestamp (x): `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE` (default: `TIMESTAMP`)
     - json (x):      `STRING`,  `RECORD` (default: `STRING`)
+    - numeric (x): `STRING`
   - **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
   - **fields (x) **: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.
   - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`)
   - **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`).
   - **description**: Description for BigQuery field
+  - **scale**: optional, [scale](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types?hl=ja#decimal_types) for numeric column (long, default is 9).
 - **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N")
 - **default_timezone**: default timezone for column_options (string, default is "UTC")
 
@@ -156,4 +158,4 @@ $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 
 ```
 $ embulk run config.yml -L PATH/embulk-output-bigquery_java/build/gemContents/
-```
+```

build.gradle

@@ -4,6 +4,7 @@ plugins {
     id "checkstyle"
     id "maven-publish"
     id "org.embulk.embulk-plugins" version "0.4.1"
+    id "com.palantir.git-version" version "0.12.3"
 }
 
 repositories {
@@ -12,9 +13,17 @@ repositories {
 }
 
 group = "io.trocco"
-version = "0.0.21"
 description = "Bigquery output for embulk"
 
+version = {
+    def vd = versionDetails()
+    if (vd.commitDistance == 0 && vd.lastTag ==~ /^v[0-9]+\.[0-9]+\.[0-9]+(\.[a-zA-Z0-9]+)?/) {
+        vd.lastTag.substring(1)
+    } else {
+        "0.0.0.${vd.gitHash}.pre"
+    }
+}()
+
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 

src/main/java/org/embulk/output/bigquery_java/config/BigqueryColumnOption.java

@@ -43,5 +43,9 @@ public interface BigqueryColumnOption extends Task {
     @ConfigDefault("null")
     public Optional<String> getDescription();
 
+    @Config("scale")
+    @ConfigDefault("9")
+    public int getScale();
+
     // TODO: fields
 }

src/main/java/org/embulk/output/bigquery_java/config/BigqueryColumnOptionType.java

@@ -8,5 +8,6 @@ public enum BigqueryColumnOptionType {
     TIMESTAMP,
     DATETIME,
     DATE,
-    RECORD
+    RECORD,
+    NUMERIC
 }

src/main/java/org/embulk/output/bigquery_java/converter/BigqueryStringConverter.java

@@ -10,6 +10,8 @@
 import org.embulk.spi.time.TimestampParseException;
 import org.embulk.spi.time.TimestampParser;
 
+import java.math.BigDecimal;
+
 public class BigqueryStringConverter {
 
     public static void convertAndSet(ObjectNode node, String name, String src, BigqueryColumnOptionType bigqueryColumnOptionType, BigqueryColumnOption columnOption) {
@@ -106,6 +108,11 @@ public static void convertAndSet(ObjectNode node, String name, String src, Bigqu
                     }
                 }
                 break;
+            case NUMERIC:
+                // Default value: 9, BigQuery NUMERIC type has a maximum scale of 9
+                int scale = columnOption != null ? columnOption.getScale() : 9;
+                node.put(name, new BigDecimal(src).setScale(scale, BigDecimal.ROUND_CEILING));
+                break;
             default:
                 throw new BigqueryNotSupportedTypeException("Invalid data convert for String");
         }

src/test/java/org/embulk/output/bigquery_java/converter/TestBigqueryStringConverter.java

@@ -14,6 +14,7 @@
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 public class TestBigqueryStringConverter {
     private ConfigSource config;
@@ -198,4 +199,23 @@ public void testConvertStringToTimestamp_withoutTimeFormat() {
         BigqueryStringConverter.convertAndSet(node, "key", "2020/05/01 00:00:00.000000 +09:00", BigqueryColumnOptionType.TIMESTAMP, columnOption);
         assertEquals("2020/05/01 00:00:00.000000 +09:00", node.get("key").asText());
     }
+
+    @Test
+    public void testConvertStringToNumeric() {
+        ObjectNode node = BigqueryUtil.getObjectMapper().createObjectNode();
+        config = loadYamlResource(embulk, "base.yml");
+        ImmutableList.Builder<ConfigSource> builder = ImmutableList.builder();
+        ConfigSource configSource = embulk.newConfig();
+        configSource.set("type", "NUMERIC");
+        configSource.set("name", "key");
+        builder.add(configSource);
+        config.set("column_options",builder.build());
+        BigqueryColumnOption columnOption = configSource.loadConfig(BigqueryColumnOption.class);
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        BigqueryStringConverter.convertAndSet(node, "key", "123.456", BigqueryColumnOptionType.NUMERIC, columnOption);
+
+        assertTrue(node.get("key").isBigDecimal());
+        assertEquals(123.456, node.get("key").asDouble(), 0);
+    }
 }