From 3569cc27b3357c3f62ccfae1670a1bf6383dee9a Mon Sep 17 00:00:00 2001 From: shollyman Date: Thu, 14 Dec 2023 11:56:15 -0700 Subject: [PATCH] feat(bigquery): add table resource tags support (#9084) This PR adds support for resource tags on BigQuery tables. Due to the nature of the feature, testing necessitates special provisioning with resourcemanager to establish a parent with the proper keys and values defined, which can then be bound to the table. While implementing, testing was done in a personal test project, with the following commands to establish the necessary tags/values: ``` gcloud resource-manager tags keys create test_tag_key --parent=projects/shollyman-testing gcloud resource-manager tags values create COFFEE --parent=tagKeys/281483438148747 gcloud resource-manager tags values create TEA --parent=tagKeys/281483438148747 gcloud resource-manager tags values create WATER --parent=tagKeys/281483438148747 ``` This integration test was used to smoke test that the feature works: ``` func TestIntegration_TableResourceTags(t *testing.T) { if client == nil { t.Skip("Integration tests skipped") } testKey := "shollyman-testing/test_tag_key" ctx := context.Background() table := dataset.Table(tableIDs.New()) resourceTags := map[string]string{ testKey: "COFFEE", } if err := table.Create(context.Background(), &TableMetadata{ Schema: schema, ResourceTags: resourceTags, }); err != nil { t.Fatalf("table.Create: %v", err) } defer table.Delete(ctx) md, err := table.Metadata(ctx) if err != nil { t.Fatalf("table.Metadata: %v", err) } var found bool for k, v := range md.ResourceTags { if k == testKey && v == "COFFEE" { found = true break } } if !found { t.Errorf("tag key/value not found") } updatedTags := map[string]string{ testKey: "COFFEE", } // Update table DefaultCollation to case-sensitive updated, err := table.Update(ctx, TableMetadataToUpdate{ ResourceTags: updatedTags, }, md.ETag) if err != nil { t.Fatalf("table.Update: %v", err) } found = false for k, v := range updated.ResourceTags { if k == testKey && v == "COFFEE" { found = true break } } if !found { t.Errorf("tag key/value not found") } } ``` I've removed the integration test for the time being after verification, given the complexity of the setup outside of the BQ service. --- bigquery/table.go | 37 +++++++++++++++++++++++++++++++++++++ bigquery/table_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/bigquery/table.go b/bigquery/table.go index b144e50d8f97..9082b0f3aa81 100644 --- a/bigquery/table.go +++ b/bigquery/table.go @@ -151,6 +151,15 @@ type TableMetadata struct { // TableConstraints contains table primary and foreign keys constraints. // Present only if the table has primary or foreign keys. TableConstraints *TableConstraints + + // The tags associated with this table. Tag + // keys are globally unique. See additional information on tags + // (https://cloud.google.com/iam/docs/tags-access-control#definitions). + // An object containing a list of "key": value pairs. The key is the + // namespaced friendly name of the tag key, e.g. "12345/environment" + // where 12345 is parent id. The value is the friendly short name of the + // tag value, e.g. "production". + ResourceTags map[string]string } // TableConstraints defines the primary key and foreign key of a table. @@ -788,6 +797,12 @@ func (tm *TableMetadata) toBQ() (*bq.Table, error) { } } } + if tm.ResourceTags != nil { + t.ResourceTags = make(map[string]string) + for k, v := range tm.ResourceTags { + t.ResourceTags[k] = v + } + } return t, nil } @@ -907,6 +922,12 @@ func bqToTableMetadata(t *bq.Table, c *Client) (*TableMetadata, error) { ForeignKeys: bqToForeignKeys(t.TableConstraints, c), } } + if t.ResourceTags != nil { + md.ResourceTags = make(map[string]string) + for k, v := range t.ResourceTags { + md.ResourceTags[k] = v + } + } return md, nil } @@ -1081,6 +1102,13 @@ func (tm *TableMetadataToUpdate) toBQ() (*bq.Table, error) { t.TableConstraints.ForceSendFields = append(t.TableConstraints.ForceSendFields, "ForeignKeys") } } + if tm.ResourceTags != nil { + t.ResourceTags = make(map[string]string) + for k, v := range tm.ResourceTags { + t.ResourceTags[k] = v + } + forceSend("ResourceTags") + } labels, forces, nulls := tm.update() t.Labels = labels t.ForceSendFields = append(t.ForceSendFields, forces...) @@ -1162,6 +1190,15 @@ type TableMetadataToUpdate struct { // such as primary and foreign keys. TableConstraints *TableConstraints + // The tags associated with this table. Tag + // keys are globally unique. See additional information on tags + // (https://cloud.google.com/iam/docs/tags-access-control#definitions). + // An object containing a list of "key": value pairs. The key is the + // namespaced friendly name of the tag key, e.g. "12345/environment" + // where 12345 is parent id. The value is the friendly short name of the + // tag value, e.g. "production". + ResourceTags map[string]string + labelUpdater } diff --git a/bigquery/table_test.go b/bigquery/table_test.go index 9a31fa19ee65..277615020152 100644 --- a/bigquery/table_test.go +++ b/bigquery/table_test.go @@ -95,6 +95,10 @@ func TestBQToTableMetadata(t *testing.T) { }, }, }, + ResourceTags: map[string]string{ + "key1": "val1", + "key2": "val2", + }, }, &TableMetadata{ Description: "desc", @@ -155,6 +159,10 @@ func TestBQToTableMetadata(t *testing.T) { }, }, }, + ResourceTags: map[string]string{ + "key1": "val1", + "key2": "val2", + }, }, }, } { @@ -188,6 +196,10 @@ func TestTableMetadataToBQ(t *testing.T) { Labels: map[string]string{"a": "b"}, ExternalDataConfig: &ExternalDataConfig{SourceFormat: Bigtable}, EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"}, + ResourceTags: map[string]string{ + "key1": "val1", + "key2": "val2", + }, }, &bq.Table{ FriendlyName: "n", @@ -201,6 +213,10 @@ func TestTableMetadataToBQ(t *testing.T) { Labels: map[string]string{"a": "b"}, ExternalDataConfiguration: &bq.ExternalDataConfiguration{SourceFormat: "BIGTABLE"}, EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"}, + ResourceTags: map[string]string{ + "key1": "val1", + "key2": "val2", + }, }, }, { @@ -511,6 +527,21 @@ func TestTableMetadataToUpdateToBQ(t *testing.T) { }, }, }, + { + tm: TableMetadataToUpdate{ + ResourceTags: map[string]string{ + "key1": "val1", + "key2": "val2", + }, + }, + want: &bq.Table{ + ResourceTags: map[string]string{ + "key1": "val1", + "key2": "val2", + }, + ForceSendFields: []string{"ResourceTags"}, + }, + }, } { got, _ := test.tm.toBQ() if !testutil.Equal(got, test.want) {