Skip to content

Commit fbe4b9b

Browse files
authored
Merge pull request #93 from trocco-io/add_column_type_numeric
Add NUMERIC Type
2 parents 6578394 + b309784 commit fbe4b9b

File tree

7 files changed

+81
-4
lines changed

7 files changed

+81
-4
lines changed

.github/workflows/gem-push.yml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
name: Ruby Gem
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- 'main'
8+
tags:
9+
- '*'
10+
pull_request:
11+
branches:
12+
- '*'
13+
types: [ opened, synchronize ]
14+
15+
jobs:
16+
build:
17+
name: Build + Publish
18+
runs-on: ubuntu-latest
19+
permissions:
20+
packages: write
21+
contents: read
22+
if: ${{ github.event_name == 'workflow_dispatch' || contains(github.ref, 'tags/v') }}
23+
steps:
24+
- uses: actions/checkout@v2
25+
- name: Set up Ruby 2.7
26+
uses: ruby/setup-ruby@v1
27+
with:
28+
ruby-version: 2.7
29+
- name: push gem
30+
uses: trocco-io/push-gem-to-gpr-action@v1
31+
with:
32+
language: java
33+
gem-path: "./build/gems/*.gem"
34+
github-token: "${{ secrets.GITHUB_TOKEN }}"

README.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,18 +116,20 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
116116

117117
- **column_options**: advanced: an array of options for columns
118118
- **name**: column name
119-
- **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, and `RECORD`. See belows for supported conversion type.
119+
- **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, `RECORD`, and `NUMERIC`. See belows for supported conversion type.
120120
- boolean (x): `BOOLEAN`, `STRING` (default: `BOOLEAN`)
121121
- long (x): `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `INTEGER`)
122122
- double (x): `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `FLOAT`)
123123
- string: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, `RECORD` (default: `STRING`)
124124
- timestamp (x): `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE` (default: `TIMESTAMP`)
125125
- json (x): `STRING`, `RECORD` (default: `STRING`)
126+
- numeric (x): `STRING`
126127
- **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
127128
- **fields (x) **: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.
128129
- **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`)
129130
- **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`).
130131
- **description**: Description for BigQuery field
132+
- **scale**: optional, [scale](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types?hl=ja#decimal_types) for numeric column (long, default is 9).
131133
- **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N")
132134
- **default_timezone**: default timezone for column_options (string, default is "UTC")
133135

@@ -156,4 +158,4 @@ $ ./gradlew gem # -t to watch change of files and rebuild continuously
156158

157159
```
158160
$ embulk run config.yml -L PATH/embulk-output-bigquery_java/build/gemContents/
159-
```
161+
```

build.gradle

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ plugins {
44
id "checkstyle"
55
id "maven-publish"
66
id "org.embulk.embulk-plugins" version "0.4.1"
7+
id "com.palantir.git-version" version "0.12.3"
78
}
89

910
repositories {
@@ -12,9 +13,17 @@ repositories {
1213
}
1314

1415
group = "io.trocco"
15-
version = "0.0.21"
1616
description = "Bigquery output for embulk"
1717

18+
version = {
19+
def vd = versionDetails()
20+
if (vd.commitDistance == 0 && vd.lastTag ==~ /^v[0-9]+\.[0-9]+\.[0-9]+(\.[a-zA-Z0-9]+)?/) {
21+
vd.lastTag.substring(1)
22+
} else {
23+
"0.0.0.${vd.gitHash}.pre"
24+
}
25+
}()
26+
1827
sourceCompatibility = 1.8
1928
targetCompatibility = 1.8
2029

src/main/java/org/embulk/output/bigquery_java/config/BigqueryColumnOption.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,5 +43,9 @@ public interface BigqueryColumnOption extends Task {
4343
@ConfigDefault("null")
4444
public Optional<String> getDescription();
4545

46+
@Config("scale")
47+
@ConfigDefault("9")
48+
public int getScale();
49+
4650
// TODO: fields
4751
}

src/main/java/org/embulk/output/bigquery_java/config/BigqueryColumnOptionType.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,6 @@ public enum BigqueryColumnOptionType {
88
TIMESTAMP,
99
DATETIME,
1010
DATE,
11-
RECORD
11+
RECORD,
12+
NUMERIC
1213
}

src/main/java/org/embulk/output/bigquery_java/converter/BigqueryStringConverter.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
import org.embulk.spi.time.TimestampParseException;
1111
import org.embulk.spi.time.TimestampParser;
1212

13+
import java.math.BigDecimal;
14+
1315
public class BigqueryStringConverter {
1416

1517
public static void convertAndSet(ObjectNode node, String name, String src, BigqueryColumnOptionType bigqueryColumnOptionType, BigqueryColumnOption columnOption) {
@@ -106,6 +108,11 @@ public static void convertAndSet(ObjectNode node, String name, String src, Bigqu
106108
}
107109
}
108110
break;
111+
case NUMERIC:
112+
// Default value: 9, BigQuery NUMERIC type has a maximum scale of 9
113+
int scale = columnOption != null ? columnOption.getScale() : 9;
114+
node.put(name, new BigDecimal(src).setScale(scale, BigDecimal.ROUND_CEILING));
115+
break;
109116
default:
110117
throw new BigqueryNotSupportedTypeException("Invalid data convert for String");
111118
}

src/test/java/org/embulk/output/bigquery_java/converter/TestBigqueryStringConverter.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.junit.Test;
1515

1616
import static org.junit.Assert.assertEquals;
17+
import static org.junit.Assert.assertTrue;
1718

1819
public class TestBigqueryStringConverter {
1920
private ConfigSource config;
@@ -198,4 +199,23 @@ public void testConvertStringToTimestamp_withoutTimeFormat() {
198199
BigqueryStringConverter.convertAndSet(node, "key", "2020/05/01 00:00:00.000000 +09:00", BigqueryColumnOptionType.TIMESTAMP, columnOption);
199200
assertEquals("2020/05/01 00:00:00.000000 +09:00", node.get("key").asText());
200201
}
202+
203+
@Test
204+
public void testConvertStringToNumeric() {
205+
ObjectNode node = BigqueryUtil.getObjectMapper().createObjectNode();
206+
config = loadYamlResource(embulk, "base.yml");
207+
ImmutableList.Builder<ConfigSource> builder = ImmutableList.builder();
208+
ConfigSource configSource = embulk.newConfig();
209+
configSource.set("type", "NUMERIC");
210+
configSource.set("name", "key");
211+
builder.add(configSource);
212+
config.set("column_options",builder.build());
213+
BigqueryColumnOption columnOption = configSource.loadConfig(BigqueryColumnOption.class);
214+
PluginTask task = config.loadConfig(PluginTask.class);
215+
216+
BigqueryStringConverter.convertAndSet(node, "key", "123.456", BigqueryColumnOptionType.NUMERIC, columnOption);
217+
218+
assertTrue(node.get("key").isBigDecimal());
219+
assertEquals(123.456, node.get("key").asDouble(), 0);
220+
}
201221
}

0 commit comments

Comments
 (0)