BigQuery: Adding support for schema autodetection

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java

@@ -61,6 +61,7 @@ public ExternalDataConfiguration apply(ExternalTableDefinition tableInfo) {
   private final Integer maxBadRecords;
   private final Boolean ignoreUnknownValues;
   private final String compression;
+  private final Boolean autodetect;
 
   public static final class Builder
       extends TableDefinition.Builder<ExternalTableDefinition, Builder> {
@@ -70,6 +71,7 @@ public static final class Builder
     private Integer maxBadRecords;
     private Boolean ignoreUnknownValues;
     private String compression;
+    private Boolean autodetect;
 
     private Builder() {
       super(Type.EXTERNAL);
@@ -82,6 +84,7 @@ private Builder(ExternalTableDefinition tableDefinition) {
       this.maxBadRecords = tableDefinition.maxBadRecords;
       this.ignoreUnknownValues = tableDefinition.ignoreUnknownValues;
       this.compression = tableDefinition.compression;
+      this.autodetect = tableDefinition.autodetect;
     }
 
     private Builder(Table tablePb) {
@@ -101,6 +104,7 @@ private Builder(Table tablePb) {
           this.formatOptions = CsvOptions.fromPb(externalDataConfiguration.getCsvOptions());
         }
         this.maxBadRecords = externalDataConfiguration.getMaxBadRecords();
+        this.autodetect = externalDataConfiguration.getAutodetect();
       }
     }
 
@@ -170,6 +174,15 @@ public Builder setCompression(String compression) {
       return this;
     }
 
+    /**
+     * [Experimental] Sets detection of schema and format options automatically. Any option specified explicitly will
+     * be honored.
+     */
+    public Builder setAutodetect(Boolean autodetect) {
+      this.autodetect = autodetect;
+      return this;
+    }
+
     /**
      * Creates an {@code ExternalTableDefinition} object.
      */
@@ -186,6 +199,7 @@ private ExternalTableDefinition(Builder builder) {
     this.maxBadRecords = builder.maxBadRecords;
     this.formatOptions = builder.formatOptions;
     this.sourceUris = builder.sourceUris;
+    this.autodetect = builder.autodetect;
   }
 
 
@@ -245,6 +259,13 @@ public <F extends FormatOptions> F getFormatOptions() {
     return (F) formatOptions;
   }
 
+  /**
+   * [Experimental] Returns whether automatic detection of schema and format options should be performed.
+   */
+  public Boolean getAutodetect() {
+    return autodetect;
+  }
+
   /**
    * Returns a builder for the {@code ExternalTableDefinition} object.
    */
@@ -260,7 +281,8 @@ ToStringHelper toStringHelper() {
         .add("formatOptions", formatOptions)
         .add("compression", compression)
         .add("ignoreUnknownValues", ignoreUnknownValues)
-        .add("maxBadRecords", maxBadRecords);
+        .add("maxBadRecords", maxBadRecords)
+        .add("autodetect", autodetect);
   }
 
   @Override
@@ -274,7 +296,7 @@ public final boolean equals(Object obj) {
   @Override
   public final int hashCode() {
     return Objects.hash(baseHashCode(), compression, ignoreUnknownValues, maxBadRecords,
-        formatOptions, sourceUris);
+        formatOptions, sourceUris, autodetect);
   }
 
   @Override
@@ -308,6 +330,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
     if (formatOptions != null && FormatOptions.CSV.equals(formatOptions.getType())) {
       externalConfigurationPb.setCsvOptions(((CsvOptions) formatOptions).toPb());
     }
+    if (autodetect != null) {
+      externalConfigurationPb.setAutodetect(autodetect);
+    }
     return externalConfigurationPb;
   }
 
@@ -417,6 +442,9 @@ static ExternalTableDefinition fromExternalDataConfiguration(
     if (externalDataConfiguration.getMaxBadRecords() != null) {
       builder.setMaxBadRecords(externalDataConfiguration.getMaxBadRecords());
     }
+    if (externalDataConfiguration.getAutodetect() != null) {
+      builder.setAutodetect(externalDataConfiguration.getAutodetect());
+    }
     return builder.build();
   }
 }

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java

@@ -101,6 +101,11 @@ interface Builder {
      */
     Builder setSchemaUpdateOptions(List<SchemaUpdateOption> schemaUpdateOptions);
 
+    /**
+     * [Experimental] Sets automatic inference of the options and schema for CSV and JSON sources.
+     */
+    Builder setAutodetect(Boolean autodetect);
+
     LoadConfiguration build();
   }
 
@@ -177,6 +182,11 @@ interface Builder {
    */
   List<SchemaUpdateOption> getSchemaUpdateOptions();
 
+  /**
+   * [Experimental] Returns whether automatic inference of the options and schema for CSV and JSON sources is set.
+   */
+  Boolean getAutodetect();
+
   /**
    * Returns a builder for the load configuration object.
    */

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java

@@ -44,6 +44,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load
   private final Schema schema;
   private final Boolean ignoreUnknownValues;
   private final List<JobInfo.SchemaUpdateOption> schemaUpdateOptions;
+  private final Boolean autodetect;
 
   public static final class Builder
       extends JobConfiguration.Builder<LoadJobConfiguration, Builder>
@@ -59,6 +60,7 @@ public static final class Builder
     private Boolean ignoreUnknownValues;
     private List<String> projectionFields;
     private List<JobInfo.SchemaUpdateOption> schemaUpdateOptions;
+    private Boolean autodetect;
 
     private Builder() {
       super(Type.LOAD);
@@ -75,6 +77,7 @@ private Builder(LoadJobConfiguration loadConfiguration) {
       this.ignoreUnknownValues = loadConfiguration.ignoreUnknownValues;
       this.sourceUris = loadConfiguration.sourceUris;
       this.schemaUpdateOptions = loadConfiguration.schemaUpdateOptions;
+      this.autodetect = loadConfiguration.autodetect;
     }
 
     private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -129,6 +132,7 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
         }
         this.schemaUpdateOptions = schemaUpdateOptionsBuilder.build();
       }
+      this.autodetect = loadConfigurationPb.getAutodetect();
     }
 
 
@@ -190,6 +194,11 @@ public Builder setSourceUris(List<String> sourceUris) {
       return this;
     }
 
+    public Builder setAutodetect(Boolean autodetect) {
+      this.autodetect = autodetect;
+      return this;
+    }
+
     @Override
     public Builder setSchemaUpdateOptions(List<JobInfo.SchemaUpdateOption> schemaUpdateOptions) {
       this.schemaUpdateOptions =
@@ -214,6 +223,7 @@ private LoadJobConfiguration(Builder builder) {
     this.schema = builder.schema;
     this.ignoreUnknownValues = builder.ignoreUnknownValues;
     this.schemaUpdateOptions = builder.schemaUpdateOptions;
+    this.autodetect = builder.autodetect;
   }
 
 
@@ -278,6 +288,10 @@ public List<String> getSourceUris() {
     return sourceUris;
   }
 
+  public Boolean getAutodetect() {
+    return autodetect;
+  }
+
   @Override
   public List<JobInfo.SchemaUpdateOption> getSchemaUpdateOptions() {
     return schemaUpdateOptions;
@@ -299,7 +313,8 @@ ToStringHelper toStringHelper() {
         .add("schema", schema)
         .add("ignoreUnknownValue", ignoreUnknownValues)
         .add("sourceUris", sourceUris)
-        .add("schemaUpdateOptions", schemaUpdateOptions);
+        .add("schemaUpdateOptions", schemaUpdateOptions)
+        .add("autodetect", autodetect);
   }
 
   @Override
@@ -363,6 +378,7 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
       }
       loadConfigurationPb.setSchemaUpdateOptions(schemaUpdateOptionsBuilder.build());
     }
+    loadConfigurationPb.setAutodetect(autodetect);
     return new com.google.api.services.bigquery.model.JobConfiguration()
         .setLoad(loadConfigurationPb);
   }

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java

@@ -47,6 +47,7 @@ public final class WriteChannelConfiguration implements LoadConfiguration, Seria
   private final Schema schema;
   private final Boolean ignoreUnknownValues;
   private final List<SchemaUpdateOption> schemaUpdateOptions;
+  private final Boolean autodetect;
 
   public static final class Builder implements LoadConfiguration.Builder {
 
@@ -58,6 +59,7 @@ public static final class Builder implements LoadConfiguration.Builder {
     private Schema schema;
     private Boolean ignoreUnknownValues;
     private List<SchemaUpdateOption> schemaUpdateOptions;
+    private Boolean autodetect;
 
     private Builder() {}
 
@@ -70,6 +72,7 @@ private Builder(WriteChannelConfiguration writeChannelConfiguration) {
       this.schema = writeChannelConfiguration.schema;
       this.ignoreUnknownValues = writeChannelConfiguration.ignoreUnknownValues;
       this.schemaUpdateOptions = writeChannelConfiguration.schemaUpdateOptions;
+      this.autodetect = writeChannelConfiguration.autodetect;
     }
 
     private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -123,6 +126,7 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
         }
         this.schemaUpdateOptions = schemaUpdateOptionsBuilder.build();
       }
+      this.autodetect = loadConfigurationPb.getAutodetect();
     }
 
 
@@ -181,6 +185,12 @@ public Builder setSchemaUpdateOptions(List<SchemaUpdateOption> schemaUpdateOptio
       return this;
     }
 
+    @Override
+    public Builder setAutodetect(Boolean autodetect) {
+      this.autodetect = autodetect;
+      return this;
+    }
+
     @Override
     public WriteChannelConfiguration build() {
       return new WriteChannelConfiguration(this);
@@ -196,6 +206,7 @@ protected WriteChannelConfiguration(Builder builder) {
     this.schema = builder.schema;
     this.ignoreUnknownValues = builder.ignoreUnknownValues;
     this.schemaUpdateOptions = builder.schemaUpdateOptions;
+    this.autodetect = builder.autodetect;
   }
 
 
@@ -257,6 +268,11 @@ public List<SchemaUpdateOption> getSchemaUpdateOptions() {
     return schemaUpdateOptions;
   }
 
+  @Override
+  public Boolean getAutodetect() {
+    return autodetect;
+  }
+
   @Override
   public Builder toBuilder() {
     return new Builder(this);
@@ -271,7 +287,8 @@ MoreObjects.ToStringHelper toStringHelper() {
         .add("maxBadRecords", maxBadRecords)
         .add("schema", schema)
         .add("ignoreUnknownValue", ignoreUnknownValues)
-        .add("schemaUpdateOptions", schemaUpdateOptions);
+        .add("schemaUpdateOptions", schemaUpdateOptions)
+        .add("autodetect", autodetect);
   }
 
   @Override
@@ -289,7 +306,7 @@ public boolean equals(Object obj) {
   @Override
   public int hashCode() {
     return Objects.hash(destinationTable, createDisposition, writeDisposition, formatOptions,
-        maxBadRecords, schema, ignoreUnknownValues, schemaUpdateOptions);
+        maxBadRecords, schema, ignoreUnknownValues, schemaUpdateOptions, autodetect);
   }
 
   WriteChannelConfiguration setProjectId(String projectId) {
@@ -336,6 +353,7 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
       }
       loadConfigurationPb.setSchemaUpdateOptions(schemaUpdateOptionsBuilder.build());
     }
+    loadConfigurationPb.setAutodetect(autodetect);
     return new com.google.api.services.bigquery.model.JobConfiguration()
         .setLoad(loadConfigurationPb);
   }

google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java

@@ -46,12 +46,14 @@ public class ExternalTableDefinitionTest {
   private static final Integer MAX_BAD_RECORDS = 42;
   private static final Boolean IGNORE_UNKNOWN_VALUES = true;
   private static final String COMPRESSION = "GZIP";
+  private static final Boolean AUTODETECT = true;
   private static final CsvOptions CSV_OPTIONS = CsvOptions.newBuilder().build();
   private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION =
       ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS)
           .setCompression(COMPRESSION)
           .setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
           .setMaxBadRecords(MAX_BAD_RECORDS)
+          .setAutodetect(AUTODETECT)
           .build();
 
   @Test
@@ -83,6 +85,7 @@ public void testBuilder() {
     assertEquals(MAX_BAD_RECORDS, EXTERNAL_TABLE_DEFINITION.getMaxBadRecords());
     assertEquals(TABLE_SCHEMA, EXTERNAL_TABLE_DEFINITION.getSchema());
     assertEquals(SOURCE_URIS, EXTERNAL_TABLE_DEFINITION.getSourceUris());
+    assertEquals(AUTODETECT, EXTERNAL_TABLE_DEFINITION.getAutodetect());
   }
 
 
@@ -106,5 +109,6 @@ private void compareExternalTableDefinition(ExternalTableDefinition expected,
     assertEquals(expected.getSchema(), value.getSchema());
     assertEquals(expected.getSourceUris(), value.getSourceUris());
     assertEquals(expected.hashCode(), value.hashCode());
+    assertEquals(expected.getAutodetect(), value.getAutodetect());
   }
 }

google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java

@@ -49,6 +49,7 @@ public class LoadJobConfigurationTest {
   private static final List<SchemaUpdateOption> SCHEMA_UPDATE_OPTIONS =
           ImmutableList.of(SchemaUpdateOption.ALLOW_FIELD_ADDITION);
   private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA);
+  private static final Boolean AUTODETECT = true;
   private static final LoadJobConfiguration LOAD_CONFIGURATION_CSV =
       LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS)
           .setCreateDisposition(CREATE_DISPOSITION)
@@ -58,6 +59,7 @@ public class LoadJobConfigurationTest {
           .setMaxBadRecords(MAX_BAD_RECORDS)
           .setSchema(TABLE_SCHEMA)
           .setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS)
+          .setAutodetect(AUTODETECT)
           .build();
   private static final DatastoreBackupOptions BACKUP_OPTIONS = DatastoreBackupOptions.newBuilder()
       .setProjectionFields(ImmutableList.of("field_1", "field_2"))
@@ -71,6 +73,7 @@ public class LoadJobConfigurationTest {
           .setMaxBadRecords(MAX_BAD_RECORDS)
           .setSchema(TABLE_SCHEMA)
           .setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS)
+          .setAutodetect(AUTODETECT)
           .build();
 
   @Test
@@ -153,6 +156,7 @@ private void compareLoadJobConfiguration(LoadJobConfiguration expected,
     assertEquals(expected.getMaxBadRecords(), value.getMaxBadRecords());
     assertEquals(expected.getSchema(), value.getSchema());
     assertEquals(expected.getDatastoreBackupOptions(), value.getDatastoreBackupOptions());
+    assertEquals(expected.getAutodetect(), value.getAutodetect());
     assertEquals(expected.getSchemaUpdateOptions(), value.getSchemaUpdateOptions());
   }
 }