Merge pull request #95 from trocco-io/add_time_partitioning_types

Add time partitioning types

Author: kekekenta

shadow-google-cloud-bigquery-helper/build.gradle

@@ -30,7 +30,7 @@ configurations {
 }
 
 dependencies {
-    compile("com.google.cloud:google-cloud-bigquery:1.106.0") {
+    compile("com.google.cloud:google-cloud-bigquery:2.14.0") {
         exclude group: "commons-logging", module: "commons-logging"
     }
 

shadow-google-cloud-bigquery-helper/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -1,40 +1,78 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
-com.fasterxml.jackson.core:jackson-core:2.10.2
-com.google.api-client:google-api-client:1.30.7
-com.google.api.grpc:proto-google-common-protos:1.17.0
-com.google.api.grpc:proto-google-iam-v1:0.13.0
-com.google.api:api-common:1.8.1
-com.google.api:gax-httpjson:0.70.1
-com.google.api:gax:1.53.1
-com.google.apis:google-api-services-bigquery:v2-rev20191211-1.30.3
-com.google.auth:google-auth-library-credentials:0.20.0
-com.google.auth:google-auth-library-oauth2-http:0.20.0
-com.google.auto.value:auto-value-annotations:1.7
-com.google.cloud:google-cloud-bigquery:1.106.0
-com.google.cloud:google-cloud-core-http:1.92.4
-com.google.cloud:google-cloud-core:1.92.4
+com.fasterxml.jackson.core:jackson-annotations:2.13.3
+com.fasterxml.jackson.core:jackson-core:2.13.3
+com.fasterxml.jackson.core:jackson-databind:2.13.3
+com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.3
+com.fasterxml.jackson:jackson-bom:2.13.3
+com.google.android:annotations:4.1.1.4
+com.google.api-client:google-api-client:1.35.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.14.2
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.138.2
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.138.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.14.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.138.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.138.2
+com.google.api.grpc:proto-google-common-protos:2.9.0
+com.google.api.grpc:proto-google-iam-v1:1.4.1
+com.google.api:api-common:2.2.1
+com.google.api:gax-grpc:2.18.2
+com.google.api:gax-httpjson:0.103.2
+com.google.api:gax:2.18.2
+com.google.apis:google-api-services-bigquery:v2-rev20220611-1.32.1
+com.google.auth:google-auth-library-credentials:1.7.0
+com.google.auth:google-auth-library-oauth2-http:1.7.0
+com.google.cloud:google-cloud-bigquery:2.14.0
+com.google.cloud:google-cloud-bigquerystorage:2.14.2
+com.google.cloud:google-cloud-core-http:2.8.0
+com.google.cloud:google-cloud-core:2.8.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.code.gson:gson:2.9.0
+com.google.errorprone:error_prone_annotations:2.14.0
+com.google.flatbuffers:flatbuffers-java:1.12.0
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:28.2-android
+com.google.guava:guava:31.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.http-client:google-http-client-appengine:1.34.1
-com.google.http-client:google-http-client-jackson2:1.34.1
-com.google.http-client:google-http-client:1.34.1
+com.google.http-client:google-http-client-apache-v2:1.42.0
+com.google.http-client:google-http-client-appengine:1.42.0
+com.google.http-client:google-http-client-gson:1.42.0
+com.google.http-client:google-http-client-jackson2:1.42.0
+com.google.http-client:google-http-client:1.42.0
 com.google.j2objc:j2objc-annotations:1.3
-com.google.oauth-client:google-oauth-client:1.30.5
-com.google.protobuf:protobuf-java-util:3.11.1
-com.google.protobuf:protobuf-java:3.11.1
-commons-codec:commons-codec:1.11
-io.grpc:grpc-context:1.22.1
-io.opencensus:opencensus-api:0.24.0
-io.opencensus:opencensus-contrib-http-util:0.24.0
+com.google.oauth-client:google-oauth-client:1.34.1
+com.google.protobuf:protobuf-java-util:3.21.1
+com.google.protobuf:protobuf-java:3.21.1
+commons-codec:commons-codec:1.15
+io.grpc:grpc-alts:1.47.0
+io.grpc:grpc-api:1.47.0
+io.grpc:grpc-auth:1.47.0
+io.grpc:grpc-context:1.47.0
+io.grpc:grpc-core:1.47.0
+io.grpc:grpc-googleapis:1.47.0
+io.grpc:grpc-grpclb:1.47.0
+io.grpc:grpc-netty-shaded:1.47.0
+io.grpc:grpc-protobuf-lite:1.47.0
+io.grpc:grpc-protobuf:1.47.0
+io.grpc:grpc-stub:1.47.0
+io.netty:netty-buffer:4.1.72.Final
+io.netty:netty-common:4.1.72.Final
+io.opencensus:opencensus-api:0.31.1
+io.opencensus:opencensus-contrib-http-util:0.31.1
+io.perfmark:perfmark-api:0.25.0
 javax.annotation:javax.annotation-api:1.3.2
-org.apache.httpcomponents:httpclient:4.5.11
-org.apache.httpcomponents:httpcore:4.4.13
+org.apache.arrow:arrow-format:8.0.0
+org.apache.arrow:arrow-memory-core:8.0.0
+org.apache.arrow:arrow-memory-netty:8.0.0
+org.apache.arrow:arrow-vector:8.0.0
+org.apache.httpcomponents:httpclient:4.5.13
+org.apache.httpcomponents:httpcore:4.4.15
 org.checkerframework:checker-compat-qual:2.5.5
+org.checkerframework:checker-qual:3.22.2
+org.codehaus.mojo:animal-sniffer-annotations:1.21
+org.conscrypt:conscrypt-openjdk-uber:2.5.1
+org.json:json:20200518
 org.slf4j:jcl-over-slf4j:1.7.12
-org.threeten:threetenbp:1.4.1
+org.slf4j:slf4j-api:1.7.25
+org.threeten:threeten-extra:1.7.0
+org.threeten:threetenbp:1.6.0

src/main/java/org/embulk/output/bigquery_java/BigqueryClient.java

@@ -131,10 +131,21 @@ public void createTableIfNotExist(String table, String dataset) {
     public TimePartitioning buildTimePartitioning(BigqueryTimePartitioning bigqueryTimePartitioning) {
         TimePartitioning.Builder timePartitioningBuilder;
 
-        if (bigqueryTimePartitioning.getType().toUpperCase().equals("DAY")) {
-            timePartitioningBuilder = TimePartitioning.newBuilder(TimePartitioning.Type.DAY);
-        } else {
-            throw new RuntimeException("time_partitioning.type is not DAY");
+        switch (bigqueryTimePartitioning.getType().toUpperCase()) {
+            case "HOUR":
+                timePartitioningBuilder = TimePartitioning.newBuilder(TimePartitioning.Type.HOUR);
+                break;
+            case "DAY":
+                timePartitioningBuilder = TimePartitioning.newBuilder(TimePartitioning.Type.DAY);
+                break;
+            case "MONTH":
+                timePartitioningBuilder = TimePartitioning.newBuilder(TimePartitioning.Type.MONTH);
+                break;
+            case "YEAR":
+                timePartitioningBuilder = TimePartitioning.newBuilder(TimePartitioning.Type.YEAR);
+                break;
+            default:
+                throw new RuntimeException("time_partitioning.type is not HOUR, DAY, MONTH, or YEAR");
         }
 
         if (bigqueryTimePartitioning.getExpirationMs().isPresent()) {

src/main/java/org/embulk/output/bigquery_java/config/BigqueryConfigValidator.java

@@ -29,8 +29,9 @@ public static void validateModeAndAutoCreteTable(PluginTask task) throws ConfigE
 
     public static void validateTimePartitioning(PluginTask task) throws ConfigException {
         if (task.getTimePartitioning().isPresent()) {
-            if (!task.getTimePartitioning().get().getType().toUpperCase().equals("DAY")) {
-                throw new ConfigException(String.format("`time_partitioning.type: %s` requires `time_partitioning.type: DAY`", task.getMode()));
+            String[] types = {"HOUR", "DAY", "MONTH", "YEAR"};
+            if (!Arrays.asList(types).contains(task.getTimePartitioning().get().getType().toUpperCase())) {
+                throw new ConfigException("time_partitioning.type: HOUR, DAY, MONTH and YEAR are supported.");
             }
         }
     }