Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: 🎸 add bigquery option of with connection #1881

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

feat: 🎸 add bigquery option of with connection #1881

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f796515
Merge pull request #1 from dataform-co/main
hiracky16 Nov 10, 2024
f069821
feat: 🎸 add bigquery option of with connection
hiracky16 Dec 15, 2024
cc443a7
feat: 🎸 add test case
hiracky16 Dec 15, 2024
f22de26
fix: 🐛 upper case
hiracky16 Dec 17, 2024
6c9e4b8
Merge branch 'dataform-co:main' into main
hiracky16 Dec 28, 2024
8454e3d
Merge branch 'main' into feature-add-option-for-with-connection
hiracky16 Dec 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion cli/api/dbadapters/execution_sql.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,12 @@ from (${query}) as insertions`;
table.bigquery && table.bigquery.clusterBy && table.bigquery.clusterBy.length > 0
? `cluster by ${table.bigquery.clusterBy.join(", ")} `
: ""
}${options.length > 0 ? `OPTIONS(${options.join(",")})` : ""}as ${table.query}`;
}${
table.bigquery && table.bigquery.withConnection
? `with connection ${table.bigquery.withConnection} `
hiracky16 marked this conversation as resolved.
Show resolved Hide resolved
: ""
}
${options.length > 0 ? `OPTIONS(${options.join(",")})` : ""}as ${table.query}`;
}

private createOrReplaceView(target: dataform.ITarget, query: string) {
Expand Down
6 changes: 6 additions & 0 deletions core/actions/incremental_table.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ interface ILegacyIncrementalTableBigqueryConfig {
labels?: { [key: string]: string };
partitionExpirationDays?: number;
requirePartitionFilter?: boolean;
withConnection?: string;
additionalOptions?: { [key: string]: string };
}

Expand Down Expand Up @@ -159,6 +160,7 @@ export class IncrementalTable extends ActionBuilder<dataform.Table> {
labels: config.labels,
partitionExpirationDays: config.partitionExpirationDays,
requirePartitionFilter: config.requirePartitionFilter,
withConnection: config.withConnection,
additionalOptions: config.additionalOptions
});
if (config.filename) {
Expand Down Expand Up @@ -488,6 +490,10 @@ export class IncrementalTable extends ActionBuilder<dataform.Table> {
unverifiedConfig.requirePartitionFilter =
unverifiedConfig.bigquery.requirePartitionFilter;
}
if (!!unverifiedConfig.bigquery.withConnection) {
unverifiedConfig.withConnection =
unverifiedConfig.bigquery.withConnection;
}
if (!!unverifiedConfig.bigquery.additionalOptions) {
unverifiedConfig.additionalOptions = unverifiedConfig.bigquery.additionalOptions;
}
Expand Down
14 changes: 14 additions & 0 deletions core/actions/table.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,11 @@ export interface IBigQueryOptions {
*/
requirePartitionFilter?: boolean;

/**
*
*/
withConnection?: string;

/**
* Key-value pairs for options [table](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#table_option_list), [view](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#view_option_list), [materialized view](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#materialized_view_option_list).
*
Expand All @@ -114,6 +119,7 @@ const IBigQueryOptionsProperties = () =>
"labels",
"partitionExpirationDays",
"requirePartitionFilter",
"withConnection",
Copy link
Contributor Author

@hiracky16 hiracky16 Dec 15, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @Ekrekr !
I am trying to add an option to the BigQuery config.
I added a new value to the config, but I am encountering a compilation error.
I am not sure what is causing it. Could you help me figure it out?

For reference, the following commands were successful:

bazel build cli  
bazel test //core/...

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not your fault - we have a couple issues with our CI/CD which we're resolving currently.

I'll get this PR in once we've managed to fix them!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for the clarification! I understand the issue is on the CI/CD side.

Please let me know if there's anything I can do to help in the meantime. I’ll wait for the fix and look forward to the PR being merged.

Thanks again for your support!

"additionalOptions"
]);

Expand Down Expand Up @@ -319,6 +325,7 @@ export class Table extends ActionBuilder<dataform.Table> {
config.requirePartitionFilter ||
config.clusterBy.length ||
Object.keys(config.labels).length ||
config.withConnection ||
Object.keys(config.additionalOptions).length
? {}
: undefined;
Expand All @@ -338,6 +345,9 @@ export class Table extends ActionBuilder<dataform.Table> {
if (Object.keys(config.labels).length) {
bigqueryOptions.labels = config.labels;
}
if (config.withConnection) {
bigqueryOptions.withConnection = config.withConnection;
}
if (Object.keys(config.additionalOptions).length) {
bigqueryOptions.additionalOptions = config.additionalOptions;
}
Expand Down Expand Up @@ -402,6 +412,7 @@ export class Table extends ActionBuilder<dataform.Table> {
config.updatePartitionFilter ||
config.clusterBy.length ||
Object.keys(config.labels).length ||
config.withConnection ||
Object.keys(config.additionalOptions).length
? {}
: undefined;
Expand All @@ -421,6 +432,9 @@ export class Table extends ActionBuilder<dataform.Table> {
if (config.clusterBy.length) {
bigqueryOptions.clusterBy = config.clusterBy;
}
if (config.withConnection) {
bigqueryOptions.withConnection = config.withConnection
}
if (Object.keys(config.labels).length) {
bigqueryOptions.labels = config.labels;
}
Expand Down
4 changes: 3 additions & 1 deletion core/main_test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1892,6 +1892,7 @@ ${exampleActionDescriptor.inputSqlxConfigBlock}
partitionExpirationDays: 1,
requirePartitionFilter: true,
clusterBy: ["clusterBy"],
withConnection: "US.external_service_connection",
labels: {"key": "val"},
additionalOptions: {
option1Key: "option1",
Expand Down Expand Up @@ -1928,7 +1929,8 @@ SELECT 1`;
},
partitionBy: "partitionBy",
partitionExpirationDays: 1,
requirePartitionFilter: true
requirePartitionFilter: true,
withConnection: "US.external_service_connection"
},
tags: ["tag1", "tag2"],
dependencyTargets: [
Expand Down
4 changes: 3 additions & 1 deletion core/session.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -585,7 +585,9 @@ export class Session {
(table.bigquery.partitionBy ||
table.bigquery.clusterBy?.length ||
table.bigquery.partitionExpirationDays ||
table.bigquery.requirePartitionFilter) &&
table.bigquery.requirePartitionFilter ||
table.bigquery.withConnection
) &&
table.enumType === dataform.TableType.VIEW &&
!table.materialized
) {
Expand Down
4 changes: 4 additions & 0 deletions protos/configs.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,8 @@ message ActionConfig {
// the action depends on data from a source which has not been declared as
// a dependency.
bool hermetic = 20;

string withConnection = 21;
}

message ViewConfig {
Expand Down Expand Up @@ -346,6 +348,8 @@ message ActionConfig {
// the action depends on data from a source which has not been declared as
// a dependency.
bool hermetic = 23;

string withConnection = 24;
}

message AssertionConfig {
Expand Down
1 change: 1 addition & 0 deletions protos/core.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ message BigQueryOptions {
int32 partition_expiration_days = 5;
bool require_partition_filter = 6;
map<string, string> additional_options = 7;
string withConnection = 8;
}

message GraphErrors {
Expand Down
67 changes: 67 additions & 0 deletions tests/api/api.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -703,6 +703,73 @@ suite("@dataform/api", () => {
);
});

test("bigquery_with_connection", () => {
const testGraph: dataform.ICompiledGraph = dataform.CompiledGraph.create({
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb", defaultLocation: "US" },
tables: [
{
target: {
schema: "schema",
name: "with_connection"
},
type: "table",
query: "select 1 as test",
bigquery: {
withConnection: "with_connection"
}
},
{
target: {
schema: "schema",
name: "plain"
},
type: "table",
query: "select 1 as test"
}
]
});
const expectedExecutionActions: dataform.IExecutionAction[] = [
{
type: "table",
tableType: "table",
target: {
schema: "schema",
name: "with_connection"
},
tasks: [
{
type: "statement",
statement:
'create or replace table `deeb.schema.additional_options` WITH CONNECTION with_connection as select 1 as test'
}
],
dependencyTargets: [],
hermeticity: dataform.ActionHermeticity.HERMETIC
},
{
type: "table",
tableType: "table",
target: {
schema: "schema",
name: "plain"
},
tasks: [
{
type: "statement",
statement: "create or replace table `deeb.schema.plain` as select 1 as test"
}
],
dependencyTargets: [],
hermeticity: dataform.ActionHermeticity.HERMETIC
}
];
const executionGraph = new Builder(testGraph, {}, dataform.WarehouseState.create({})).build();
expect(asPlainObject(executionGraph.actions)).deep.equals(
asPlainObject(expectedExecutionActions)
);
});
});

test("bigquery_additional_options", () => {
const testGraph: dataform.ICompiledGraph = dataform.CompiledGraph.create({
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb", defaultLocation: "US" },
Expand Down