feat: publish new fields for CMEK (#222)

* changes without context

        autosynth cannot find the source of changes triggered by earlier changes in this
        repository, or by version upgrades to tools such as linters.

* Bump gapic-generator to 2.6.1. - Fix a scenario where generator attempts to assign a string to an integer in tests by using a separate value generator in test generation

PiperOrigin-RevId: 336931287

Source-Author: Google APIs <noreply@google.com>
Source-Date: Tue Oct 13 12:29:21 2020 -0700
Source-Repo: googleapis/googleapis
Source-Sha: 3a935fab757e09c72afd4aa121147a4c97dccc3e
Source-Link: googleapis/googleapis@3a935fa

* chore: update grpc dependency to v1.33.1

PiperOrigin-RevId: 338646463

Source-Author: Google APIs <noreply@google.com>
Source-Date: Fri Oct 23 03:57:15 2020 -0700
Source-Repo: googleapis/googleapis
Source-Sha: 20b11dfe4538cd5da7b4c3dd7d2bf5b9922ff3ed
Source-Link: googleapis/googleapis@20b11df

* feat:Update BigtableTableAdmin GetIamPolicy to include the additional binding for Backup. feat:Change DeleteAppProfileRequest.ignore_warnings to REQUIRED.

PiperOrigin-RevId: 339464550

Source-Author: Google APIs <noreply@google.com>
Source-Date: Wed Oct 28 08:32:48 2020 -0700
Source-Repo: googleapis/googleapis
Source-Sha: ccd6462d31e6422fd188b6590aa8d0ad03e7d9a3
Source-Link: googleapis/googleapis@ccd6462

* feat: migrate bigtable retry/timeout settings to gRPC's service configs

Committer: @miraleung
PiperOrigin-RevId: 346894665

Source-Author: Google APIs <noreply@google.com>
Source-Date: Thu Dec 10 16:55:31 2020 -0800
Source-Repo: googleapis/googleapis
Source-Sha: cbbd3170bcf217e36ae72f4ac522449bf861346f
Source-Link: googleapis/googleapis@cbbd317

* chore: migrate bigtable to the Python microgenerator

PiperOrigin-RevId: 356992836

Source-Author: Google APIs <noreply@google.com>
Source-Date: Thu Feb 11 09:33:53 2021 -0800
Source-Repo: googleapis/googleapis
Source-Sha: 1e0c7413684ca6f6322620ecfc0d3e0352933dc1
Source-Link: googleapis/googleapis@1e0c741

* chore: migrate StreetView to the {Java,Python} microgenerators

Committer: @miraleung
PiperOrigin-RevId: 357863594

Source-Author: Google APIs <noreply@google.com>
Source-Date: Tue Feb 16 20:19:58 2021 -0800
Source-Repo: googleapis/googleapis
Source-Sha: 8ca872ced04c96c05a36db3d2113d568ac814be8
Source-Link: googleapis/googleapis@8ca872c

* feat: Publish new fields to support Customer Managed Encryption Keys (CMEK) on the existing Cloud Bigtable service methods.

PiperOrigin-RevId: 359130387

Source-Author: Google APIs <noreply@google.com>
Source-Date: Tue Feb 23 14:08:20 2021 -0800
Source-Repo: googleapis/googleapis
Source-Sha: eabec5a21219401bad79e1cc7d900c1658aee5fd
Source-Link: googleapis/googleapis@eabec5a

* fix: Use rules_gapic to v0.5.0. Fixes handling parameters with spaces.

Committer: @alexander-fenster
PiperOrigin-RevId: 359364666

Source-Author: Google APIs <noreply@google.com>
Source-Date: Wed Feb 24 14:01:05 2021 -0800
Source-Repo: googleapis/googleapis
Source-Sha: c06bbe28cc7287a55bf7926ee48da2565854de7f
Source-Link: googleapis/googleapis@c06bbe2

* fix noxfile

Co-authored-by: Kristen O'Leary <kaoleary@google.com>

Author: yoshi-automation

packages/google-cloud-bigtable/.gitignore

@@ -50,10 +50,8 @@ docs.metadata
 
 # Virtual environment
 env/
-
-# Test logs
 coverage.xml
-*sponge_log.xml
+sponge_log.xml
 
 # System test environment variables.
 system_tests/local_test_setup

packages/google-cloud-bigtable/.kokoro/build.sh

@@ -40,16 +40,6 @@ python3 -m pip uninstall --yes --quiet nox-automation
 python3 -m pip install --upgrade --quiet nox
 python3 -m nox --version
 
-# If this is a continuous build, send the test log to the FlakyBot.
-# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
-if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then
-  cleanup() {
-    chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
-    $KOKORO_GFILE_DIR/linux_amd64/flakybot
-  }
-  trap cleanup EXIT HUP
-fi
-
 # If NOX_SESSION is set, it only runs the specified session,
 # otherwise run all the sessions.
 if [[ -n "${NOX_SESSION:-}" ]]; then

packages/google-cloud-bigtable/google/cloud/bigtable_admin_v2/__init__.py

@@ -76,6 +76,7 @@
 from .types.table import Backup
 from .types.table import BackupInfo
 from .types.table import ColumnFamily
+from .types.table import EncryptionInfo
 from .types.table import GcRule
 from .types.table import RestoreInfo
 from .types.table import RestoreSourceType
@@ -87,7 +88,7 @@
     "AppProfile",
     "Backup",
     "BackupInfo",
-    "BigtableInstanceAdminClient",
+    "BigtableTableAdminClient",
     "CheckConsistencyRequest",
     "CheckConsistencyResponse",
     "Cluster",
@@ -109,6 +110,7 @@
     "DeleteSnapshotRequest",
     "DeleteTableRequest",
     "DropRowRangeRequest",
+    "EncryptionInfo",
     "GcRule",
     "GenerateConsistencyTokenRequest",
     "GenerateConsistencyTokenResponse",
@@ -149,5 +151,5 @@
     "UpdateBackupRequest",
     "UpdateClusterMetadata",
     "UpdateInstanceMetadata",
-    "BigtableTableAdminClient",
+    "BigtableInstanceAdminClient",
 )

packages/google-cloud-bigtable/google/cloud/bigtable_admin_v2/proto/instance.proto

@@ -1,4 +1,4 @@
-// Copyright 2019 Google LLC.
+// Copyright 2021 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 
 syntax = "proto3";
 
@@ -28,6 +27,10 @@ option java_outer_classname = "InstanceProto";
 option java_package = "com.google.bigtable.admin.v2";
 option php_namespace = "Google\\Cloud\\Bigtable\\Admin\\V2";
 option ruby_package = "Google::Cloud::Bigtable::Admin::V2";
+option (google.api.resource_definition) = {
+  type: "cloudkms.googleapis.com/CryptoKey"
+  pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}"
+};
 
 // A collection of Bigtable [Tables][google.bigtable.admin.v2.Table] and
 // the resources that serve them.
@@ -113,6 +116,22 @@ message Cluster {
     pattern: "projects/{project}/instances/{instance}/clusters/{cluster}"
   };
 
+  // Cloud Key Management Service (Cloud KMS) settings for a CMEK-protected
+  // cluster.
+  message EncryptionConfig {
+    // Describes the Cloud KMS encryption key that will be used to protect the
+    // destination Bigtable cluster. The requirements for this key are:
+    //  1) The Cloud Bigtable service account associated with the project that
+    //  contains this cluster must be granted the
+    //  `cloudkms.cryptoKeyEncrypterDecrypter` role on the CMEK key.
+    //  2) Only regional keys can be used and the region of the CMEK key must
+    //  match the region of the cluster.
+    // 3) All clusters within an instance must use the same CMEK key.
+    string kms_key_name = 1 [(google.api.resource_reference) = {
+      type: "cloudkms.googleapis.com/CryptoKey"
+    }];
+  }
+
   // Possible states of a cluster.
   enum State {
     // The state of the cluster could not be determined.
@@ -162,6 +181,10 @@ message Cluster {
   // The type of storage used by this cluster to serve its
   // parent instance's tables, unless explicitly overridden.
   StorageType default_storage_type = 5;
+
+  // Immutable. The encryption configuration for CMEK-protected clusters.
+  EncryptionConfig encryption_config = 6
+      [(google.api.field_behavior) = IMMUTABLE];
 }
 
 // A configuration object describing how Cloud Bigtable should treat traffic
@@ -194,7 +217,7 @@ message AppProfile {
 
   // (`OutputOnly`)
   // The unique name of the app profile. Values are of the form
-  // `projects/<project>/instances/<instance>/appProfiles/[_a-zA-Z0-9][-_.a-zA-Z0-9]*`.
+  // `projects/{project}/instances/{instance}/appProfiles/[_a-zA-Z0-9][-_.a-zA-Z0-9]*`.
   string name = 1;
 
   // Strongly validated etag for optimistic concurrency control. Preserve the

packages/google-cloud-bigtable/google/cloud/bigtable_admin_v2/proto/table.proto

@@ -1,4 +1,4 @@
-// Copyright 2020 Google LLC
+// Copyright 2021 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import "google/api/field_behavior.proto";
 import "google/api/resource.proto";
 import "google/protobuf/duration.proto";
 import "google/protobuf/timestamp.proto";
+import "google/rpc/status.proto";
 
 option csharp_namespace = "Google.Cloud.Bigtable.Admin.V2";
 option go_package = "google.golang.org/genproto/googleapis/bigtable/admin/v2;admin";
@@ -28,6 +29,10 @@ option java_outer_classname = "TableProto";
 option java_package = "com.google.bigtable.admin.v2";
 option php_namespace = "Google\\Cloud\\Bigtable\\Admin\\V2";
 option ruby_package = "Google::Cloud::Bigtable::Admin::V2";
+option (google.api.resource_definition) = {
+  type: "cloudkms.googleapis.com/CryptoKeyVersion"
+  pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}/cryptoKeyVersions/{crypto_key_version}"
+};
 
 // Indicates the type of the restore source.
 enum RestoreSourceType {
@@ -92,6 +97,14 @@ message Table {
 
     // Output only. The state of replication for the table in this cluster.
     ReplicationState replication_state = 1;
+
+    // Output only. The encryption information for the table in this cluster.
+    // If the encryption key protecting this resource is customer managed, then
+    // its version can be rotated in Cloud Key Management Service (Cloud KMS).
+    // The primary version of the key and its status will be reflected here when
+    // changes propagate from Cloud KMS.
+    repeated EncryptionInfo encryption_info = 2
+        [(google.api.field_behavior) = OUTPUT_ONLY];
   }
 
   // Possible timestamp granularities to use when keeping multiple versions
@@ -120,20 +133,23 @@ message Table {
     // state.
     REPLICATION_VIEW = 3;
 
+    // Only populates 'name' and fields related to the table's encryption state.
+    ENCRYPTION_VIEW = 5;
+
     // Populates all fields.
     FULL = 4;
   }
 
-  // Output only. The unique name of the table. Values are of the form
-  // `projects/<project>/instances/<instance>/tables/[_a-zA-Z0-9][-_.a-zA-Z0-9]*`.
+  // The unique name of the table. Values are of the form
+  // `projects/{project}/instances/{instance}/tables/[_a-zA-Z0-9][-_.a-zA-Z0-9]*`.
   // Views: `NAME_ONLY`, `SCHEMA_VIEW`, `REPLICATION_VIEW`, `FULL`
   string name = 1;
 
   // Output only. Map from cluster ID to per-cluster table state.
   // If it could not be determined whether or not the table has data in a
   // particular cluster (for example, if its zone is unavailable), then
   // there will be an entry for the cluster with UNKNOWN `replication_status`.
-  // Views: `REPLICATION_VIEW`, `FULL`
+  // Views: `REPLICATION_VIEW`, `ENCRYPTION_VIEW`, `FULL`
   map<string, ClusterState> cluster_states = 2;
 
   // (`CreationOnly`)
@@ -196,6 +212,51 @@ message GcRule {
   }
 }
 
+// Encryption information for a given resource.
+// If this resource is protected with customer managed encryption, the in-use
+// Cloud Key Management Service (Cloud KMS) key version is specified along with
+// its status.
+message EncryptionInfo {
+  // Possible encryption types for a resource.
+  enum EncryptionType {
+    // Encryption type was not specified, though data at rest remains encrypted.
+    ENCRYPTION_TYPE_UNSPECIFIED = 0;
+
+    // The data backing this resource is encrypted at rest with a key that is
+    // fully managed by Google. No key version or status will be populated.
+    // This is the default state.
+    GOOGLE_DEFAULT_ENCRYPTION = 1;
+
+    // The data backing this resource is encrypted at rest with a key that is
+    // managed by the customer.
+    // The in-use version of the key and its status are populated for
+    // CMEK-protected tables.
+    // CMEK-protected backups are pinned to the key version that was in use at
+    // the time the backup was taken. This key version is populated but its
+    // status is not tracked and is reported as `UNKNOWN`.
+    CUSTOMER_MANAGED_ENCRYPTION = 2;
+  }
+
+  // Output only. The type of encryption used to protect this resource.
+  EncryptionType encryption_type = 3
+      [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. The status of encrypt/decrypt calls on underlying data for
+  // this resource. Regardless of status, the existing data is always encrypted
+  // at rest.
+  google.rpc.Status encryption_status = 4
+      [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. The version of the Cloud KMS key specified in the parent
+  // cluster that is in use for the data underlying this table.
+  string kms_key_version = 2 [
+    (google.api.field_behavior) = OUTPUT_ONLY,
+    (google.api.resource_reference) = {
+      type: "cloudkms.googleapis.com/CryptoKeyVersion"
+    }
+  ];
+}
+
 // A snapshot of a table at a particular time. A snapshot can be used as a
 // checkpoint for data restoration or a data source for a new table.
 //
@@ -225,7 +286,7 @@ message Snapshot {
 
   // Output only. The unique name of the snapshot.
   // Values are of the form
-  // `projects/<project>/instances/<instance>/clusters/<cluster>/snapshots/<snapshot>`.
+  // `projects/{project}/instances/{instance}/clusters/{cluster}/snapshots/{snapshot}`.
   string name = 1;
 
   // Output only. The source table at the time the snapshot was taken.
@@ -318,6 +379,10 @@ message Backup {
 
   // Output only. The current state of the backup.
   State state = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. The encryption information for the backup.
+  EncryptionInfo encryption_info = 9
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // Information about a backup.

packages/google-cloud-bigtable/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py

@@ -62,6 +62,10 @@ class BigtableInstanceAdminAsyncClient:
     )
     cluster_path = staticmethod(BigtableInstanceAdminClient.cluster_path)
     parse_cluster_path = staticmethod(BigtableInstanceAdminClient.parse_cluster_path)
+    crypto_key_path = staticmethod(BigtableInstanceAdminClient.crypto_key_path)
+    parse_crypto_key_path = staticmethod(
+        BigtableInstanceAdminClient.parse_crypto_key_path
+    )
     instance_path = staticmethod(BigtableInstanceAdminClient.instance_path)
     parse_instance_path = staticmethod(BigtableInstanceAdminClient.parse_instance_path)
 

packages/google-cloud-bigtable/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py

@@ -201,6 +201,27 @@ def parse_cluster_path(path: str) -> Dict[str, str]:
         )
         return m.groupdict() if m else {}
 
+    @staticmethod
+    def crypto_key_path(
+        project: str, location: str, key_ring: str, crypto_key: str,
+    ) -> str:
+        """Return a fully-qualified crypto_key string."""
+        return "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}".format(
+            project=project,
+            location=location,
+            key_ring=key_ring,
+            crypto_key=crypto_key,
+        )
+
+    @staticmethod
+    def parse_crypto_key_path(path: str) -> Dict[str, str]:
+        """Parse a crypto_key path into its component segments."""
+        m = re.match(
+            r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/keyRings/(?P<key_ring>.+?)/cryptoKeys/(?P<crypto_key>.+?)$",
+            path,
+        )
+        return m.groupdict() if m else {}
+
     @staticmethod
     def instance_path(project: str, instance: str,) -> str:
         """Return a fully-qualified instance string."""

packages/google-cloud-bigtable/google/cloud/bigtable_admin_v2/services/bigtable_table_admin/async_client.py

@@ -61,6 +61,12 @@ class BigtableTableAdminAsyncClient:
     parse_backup_path = staticmethod(BigtableTableAdminClient.parse_backup_path)
     cluster_path = staticmethod(BigtableTableAdminClient.cluster_path)
     parse_cluster_path = staticmethod(BigtableTableAdminClient.parse_cluster_path)
+    crypto_key_version_path = staticmethod(
+        BigtableTableAdminClient.crypto_key_version_path
+    )
+    parse_crypto_key_version_path = staticmethod(
+        BigtableTableAdminClient.parse_crypto_key_version_path
+    )
     instance_path = staticmethod(BigtableTableAdminClient.instance_path)
     parse_instance_path = staticmethod(BigtableTableAdminClient.parse_instance_path)
     snapshot_path = staticmethod(BigtableTableAdminClient.snapshot_path)

packages/google-cloud-bigtable/google/cloud/bigtable_admin_v2/services/bigtable_table_admin/client.py

@@ -202,6 +202,32 @@ def parse_cluster_path(path: str) -> Dict[str, str]:
         )
         return m.groupdict() if m else {}
 
+    @staticmethod
+    def crypto_key_version_path(
+        project: str,
+        location: str,
+        key_ring: str,
+        crypto_key: str,
+        crypto_key_version: str,
+    ) -> str:
+        """Return a fully-qualified crypto_key_version string."""
+        return "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}/cryptoKeyVersions/{crypto_key_version}".format(
+            project=project,
+            location=location,
+            key_ring=key_ring,
+            crypto_key=crypto_key,
+            crypto_key_version=crypto_key_version,
+        )
+
+    @staticmethod
+    def parse_crypto_key_version_path(path: str) -> Dict[str, str]:
+        """Parse a crypto_key_version path into its component segments."""
+        m = re.match(
+            r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/keyRings/(?P<key_ring>.+?)/cryptoKeys/(?P<crypto_key>.+?)/cryptoKeyVersions/(?P<crypto_key_version>.+?)$",
+            path,
+        )
+        return m.groupdict() if m else {}
+
     @staticmethod
     def instance_path(project: str, instance: str,) -> str:
         """Return a fully-qualified instance string."""

packages/google-cloud-bigtable/google/cloud/bigtable_admin_v2/types/__init__.py

@@ -53,6 +53,7 @@
     Table,
     ColumnFamily,
     GcRule,
+    EncryptionInfo,
     Snapshot,
     Backup,
     BackupInfo,
@@ -122,6 +123,7 @@
     "Table",
     "ColumnFamily",
     "GcRule",
+    "EncryptionInfo",
     "Snapshot",
     "Backup",
     "BackupInfo",