From 8bb0e9454c80699af4341c34a5cd56a3bd21dfbe Mon Sep 17 00:00:00 2001 From: arithmetic1728 <58957152+arithmetic1728@users.noreply.github.com> Date: Wed, 16 Sep 2020 11:17:33 -0700 Subject: [PATCH] feat: regenerate client lib to pick up new mtls env (#55) * feat: regenerate client lib to pick up new mtls env * update google-api-core and proto-plus version --- .../google-cloud-kms/.github/snippet-bot.yml | 0 .../.kokoro/populate-secrets.sh | 43 ++ .../.kokoro/release/common.cfg | 50 +- .../google-cloud-kms/.kokoro/trampoline.sh | 15 +- packages/google-cloud-kms/docs/conf.py | 2 +- .../key_management_service/async_client.py | 76 ++- .../services/key_management_service/client.py | 70 ++- .../key_management_service/transports/base.py | 42 +- .../key_management_service/transports/grpc.py | 46 +- .../transports/grpc_asyncio.py | 51 +- packages/google-cloud-kms/noxfile.py | 4 +- .../samples/snippets/README.md | 1 + .../scripts/decrypt-secrets.sh | 15 +- packages/google-cloud-kms/setup.py | 4 +- packages/google-cloud-kms/synth.metadata | 14 +- .../kms_v1/test_key_management_service.py | 570 ++++++++++-------- 16 files changed, 573 insertions(+), 430 deletions(-) create mode 100644 packages/google-cloud-kms/.github/snippet-bot.yml create mode 100755 packages/google-cloud-kms/.kokoro/populate-secrets.sh diff --git a/packages/google-cloud-kms/.github/snippet-bot.yml b/packages/google-cloud-kms/.github/snippet-bot.yml new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/google-cloud-kms/.kokoro/populate-secrets.sh b/packages/google-cloud-kms/.kokoro/populate-secrets.sh new file mode 100755 index 000000000000..f52514257ef0 --- /dev/null +++ b/packages/google-cloud-kms/.kokoro/populate-secrets.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2020 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} +function msg { println "$*" >&2 ;} +function println { printf '%s\n' "$(now) $*" ;} + + +# Populates requested secrets set in SECRET_MANAGER_KEYS from service account: +# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com +SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" +msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" +mkdir -p ${SECRET_LOCATION} +for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") +do + msg "Retrieving secret ${key}" + docker run --entrypoint=gcloud \ + --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \ + gcr.io/google.com/cloudsdktool/cloud-sdk \ + secrets versions access latest \ + --project cloud-devrel-kokoro-resources \ + --secret ${key} > \ + "${SECRET_LOCATION}/${key}" + if [[ $? == 0 ]]; then + msg "Secret written to ${SECRET_LOCATION}/${key}" + else + msg "Error retrieving secret ${key}" + fi +done diff --git a/packages/google-cloud-kms/.kokoro/release/common.cfg b/packages/google-cloud-kms/.kokoro/release/common.cfg index 653e734869ea..3be661ae4ae4 100644 --- a/packages/google-cloud-kms/.kokoro/release/common.cfg +++ b/packages/google-cloud-kms/.kokoro/release/common.cfg @@ -23,42 +23,18 @@ env_vars: { value: "github/python-kms/.kokoro/release.sh" } -# Fetch the token needed for reporting release status to GitHub -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "yoshi-automation-github-key" - } - } -} - -# Fetch PyPI password -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "google_cloud_pypi_password" - } - } -} - -# Fetch magictoken to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "releasetool-magictoken" - } - } +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google_cloud_pypi_password" + } + } } -# Fetch api key to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "magic-github-proxy-api-key" - } - } -} +# Tokens needed to report release status back to GitHub +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" +} \ No newline at end of file diff --git a/packages/google-cloud-kms/.kokoro/trampoline.sh b/packages/google-cloud-kms/.kokoro/trampoline.sh index e8c4251f3ed4..f39236e943a8 100755 --- a/packages/google-cloud-kms/.kokoro/trampoline.sh +++ b/packages/google-cloud-kms/.kokoro/trampoline.sh @@ -15,9 +15,14 @@ set -eo pipefail -python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" || ret_code=$? +# Always run the cleanup script, regardless of the success of bouncing into +# the container. +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT -chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh -${KOKORO_GFILE_DIR}/trampoline_cleanup.sh || true - -exit ${ret_code} +$(dirname $0)/populate-secrets.sh # Secret Manager secrets. +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" \ No newline at end of file diff --git a/packages/google-cloud-kms/docs/conf.py b/packages/google-cloud-kms/docs/conf.py index 12936d0dd67d..449f5a4f3ab3 100644 --- a/packages/google-cloud-kms/docs/conf.py +++ b/packages/google-cloud-kms/docs/conf.py @@ -29,7 +29,7 @@ # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = "1.6.3" +needs_sphinx = "1.5.5" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom diff --git a/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/async_client.py b/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/async_client.py index 21bfb4d82e97..feb6e574fb3a 100644 --- a/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/async_client.py +++ b/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/async_client.py @@ -64,14 +64,21 @@ class KeyManagementServiceAsyncClient: DEFAULT_MTLS_ENDPOINT = KeyManagementServiceClient.DEFAULT_MTLS_ENDPOINT crypto_key_path = staticmethod(KeyManagementServiceClient.crypto_key_path) - - import_job_path = staticmethod(KeyManagementServiceClient.import_job_path) - + parse_crypto_key_path = staticmethod( + KeyManagementServiceClient.parse_crypto_key_path + ) crypto_key_version_path = staticmethod( KeyManagementServiceClient.crypto_key_version_path ) - + parse_crypto_key_version_path = staticmethod( + KeyManagementServiceClient.parse_crypto_key_version_path + ) + import_job_path = staticmethod(KeyManagementServiceClient.import_job_path) + parse_import_job_path = staticmethod( + KeyManagementServiceClient.parse_import_job_path + ) key_ring_path = staticmethod(KeyManagementServiceClient.key_ring_path) + parse_key_ring_path = staticmethod(KeyManagementServiceClient.parse_key_ring_path) from_service_account_file = KeyManagementServiceClient.from_service_account_file from_service_account_json = from_service_account_file @@ -104,16 +111,19 @@ def __init__( client_options (ClientOptions): Custom options for the client. It won't take effect if a ``transport`` instance is provided. (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT environment variable can also be used to override the endpoint: "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint, this is the default value for - the environment variable) and "auto" (auto switch to the default - mTLS endpoint if client SSL credentials is present). However, - the ``api_endpoint`` property takes precedence if provided. - (2) The ``client_cert_source`` property is used to provide client - SSL credentials for mutual TLS transport. If not provided, the - default SSL credentials will be used if present. + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. Raises: google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport @@ -192,9 +202,9 @@ async def list_key_rings( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -284,9 +294,9 @@ async def list_crypto_keys( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -377,9 +387,9 @@ async def list_crypto_key_versions( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -469,9 +479,9 @@ async def list_import_jobs( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -559,9 +569,9 @@ async def get_key_ring( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -652,9 +662,9 @@ async def get_crypto_key( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -750,9 +760,9 @@ async def get_crypto_key_version( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -842,9 +852,9 @@ async def get_public_key( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -969,9 +979,9 @@ async def get_import_job( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -1072,9 +1082,9 @@ async def create_key_ring( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -1185,9 +1195,9 @@ async def create_crypto_key( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -1508,9 +1518,9 @@ async def create_import_job( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -1606,9 +1616,9 @@ async def update_crypto_key( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -1726,9 +1736,9 @@ async def update_crypto_key_version( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -1839,9 +1849,9 @@ async def encrypt( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -1936,9 +1946,9 @@ async def decrypt( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -2035,9 +2045,9 @@ async def asymmetric_sign( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -2134,9 +2144,9 @@ async def asymmetric_decrypt( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -2237,9 +2247,9 @@ async def update_crypto_key_primary_version( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -2353,9 +2363,9 @@ async def destroy_crypto_key_version( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -2460,9 +2470,9 @@ async def restore_crypto_key_version( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, diff --git a/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/client.py b/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/client.py index ebcc619310eb..8e6aff4f2bec 100644 --- a/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/client.py +++ b/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/client.py @@ -16,6 +16,7 @@ # from collections import OrderedDict +from distutils import util import os import re from typing import Callable, Dict, Sequence, Tuple, Type, Union @@ -27,6 +28,7 @@ from google.api_core import retry as retries # type: ignore from google.auth import credentials # type: ignore from google.auth.transport import mtls # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore from google.auth.exceptions import MutualTLSChannelError # type: ignore from google.oauth2 import service_account # type: ignore @@ -259,16 +261,19 @@ def __init__( client_options (ClientOptions): Custom options for the client. It won't take effect if a ``transport`` instance is provided. (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT environment variable can also be used to override the endpoint: "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint, this is the default value for - the environment variable) and "auto" (auto switch to the default - mTLS endpoint if client SSL credentials is present). However, - the ``api_endpoint`` property takes precedence if provided. - (2) The ``client_cert_source`` property is used to provide client - SSL credentials for mutual TLS transport. If not provided, the - default SSL credentials will be used if present. + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. @@ -284,25 +289,43 @@ def __init__( if client_options is None: client_options = ClientOptions.ClientOptions() - if client_options.api_endpoint is None: - use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS", "never") + # Create SSL credentials for mutual TLS if needed. + use_client_cert = bool( + util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false")) + ) + + ssl_credentials = None + is_mtls = False + if use_client_cert: + if client_options.client_cert_source: + import grpc # type: ignore + + cert, key = client_options.client_cert_source() + ssl_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + is_mtls = True + else: + creds = SslCredentials() + is_mtls = creds.is_mtls + ssl_credentials = creds.ssl_credentials if is_mtls else None + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + else: + use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") if use_mtls_env == "never": - client_options.api_endpoint = self.DEFAULT_ENDPOINT + api_endpoint = self.DEFAULT_ENDPOINT elif use_mtls_env == "always": - client_options.api_endpoint = self.DEFAULT_MTLS_ENDPOINT + api_endpoint = self.DEFAULT_MTLS_ENDPOINT elif use_mtls_env == "auto": - has_client_cert_source = ( - client_options.client_cert_source is not None - or mtls.has_default_client_cert_source() - ) - client_options.api_endpoint = ( - self.DEFAULT_MTLS_ENDPOINT - if has_client_cert_source - else self.DEFAULT_ENDPOINT + api_endpoint = ( + self.DEFAULT_MTLS_ENDPOINT if is_mtls else self.DEFAULT_ENDPOINT ) else: raise MutualTLSChannelError( - "Unsupported GOOGLE_API_USE_MTLS value. Accepted values: never, auto, always" + "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted values: never, auto, always" ) # Save or instantiate the transport. @@ -326,10 +349,9 @@ def __init__( self._transport = Transport( credentials=credentials, credentials_file=client_options.credentials_file, - host=client_options.api_endpoint, + host=api_endpoint, scopes=client_options.scopes, - api_mtls_endpoint=client_options.api_endpoint, - client_cert_source=client_options.client_cert_source, + ssl_channel_credentials=ssl_credentials, quota_project_id=client_options.quota_project_id, client_info=client_info, ) diff --git a/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/base.py b/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/base.py index d678a926564e..43cfc0a5f251 100644 --- a/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/base.py +++ b/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/base.py @@ -118,9 +118,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -133,9 +133,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -148,9 +148,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -163,9 +163,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -178,9 +178,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -193,9 +193,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -208,9 +208,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -223,9 +223,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -238,9 +238,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -253,9 +253,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -268,9 +268,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -293,9 +293,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -308,9 +308,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -323,9 +323,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -338,9 +338,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -353,9 +353,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -368,9 +368,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -383,9 +383,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -398,9 +398,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -413,9 +413,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, @@ -428,9 +428,9 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.InternalServerError, exceptions.ServiceUnavailable, - exceptions.DeadlineExceeded, ), ), default_timeout=60.0, diff --git a/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/grpc.py b/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/grpc.py index c5da6fc929ac..1992c5fa442a 100644 --- a/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/grpc.py +++ b/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/grpc.py @@ -15,6 +15,7 @@ # limitations under the License. # +import warnings from typing import Callable, Dict, Optional, Sequence, Tuple from google.api_core import grpc_helpers # type: ignore @@ -23,7 +24,6 @@ from google.auth import credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore - import grpc # type: ignore from google.cloud.kms_v1.types import resources @@ -70,6 +70,7 @@ def __init__( channel: grpc.Channel = None, api_mtls_endpoint: str = None, client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, + ssl_channel_credentials: grpc.ChannelCredentials = None, quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, ) -> None: @@ -90,14 +91,16 @@ def __init__( ignored if ``channel`` is provided. channel (Optional[grpc.Channel]): A ``Channel`` instance through which to make calls. - api_mtls_endpoint (Optional[str]): The mutual TLS endpoint. If - provided, it overrides the ``host`` argument and tries to create + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create a mutual TLS channel with client SSL credentials from ``client_cert_source`` or applicatin default SSL credentials. - client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): A - callback to provide client SSL certificate bytes and private key - bytes, both in PEM format. It is ignored if ``api_mtls_endpoint`` - is None. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for grpc channel. It is ignored if ``channel`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): @@ -120,6 +123,11 @@ def __init__( # If a channel was explicitly provided, set it. self._grpc_channel = channel elif api_mtls_endpoint: + warnings.warn( + "api_mtls_endpoint and client_cert_source are deprecated", + DeprecationWarning, + ) + host = ( api_mtls_endpoint if ":" in api_mtls_endpoint @@ -150,6 +158,23 @@ def __init__( scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, ) + else: + host = host if ":" in host else host + ":443" + + if credentials is None: + credentials, _ = auth.default( + scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id + ) + + # create a new channel. The provided one is ignored. + self._grpc_channel = type(self).create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + ssl_credentials=ssl_channel_credentials, + scopes=scopes or self.AUTH_SCOPES, + quota_project_id=quota_project_id, + ) self._stubs = {} # type: Dict[str, Callable] @@ -215,13 +240,6 @@ def grpc_channel(self) -> grpc.Channel: This property caches on the instance; repeated calls return the same channel. """ - # Sanity check: Only create a new channel if we do not already - # have one. - if not hasattr(self, "_grpc_channel"): - self._grpc_channel = self.create_channel( - self._host, credentials=self._credentials, - ) - # Return the channel from cache. return self._grpc_channel diff --git a/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/grpc_asyncio.py b/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/grpc_asyncio.py index 56c887b31829..57692915e267 100644 --- a/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/grpc_asyncio.py +++ b/packages/google-cloud-kms/google/cloud/kms_v1/services/key_management_service/transports/grpc_asyncio.py @@ -15,10 +15,12 @@ # limitations under the License. # +import warnings from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple from google.api_core import gapic_v1 # type: ignore from google.api_core import grpc_helpers_async # type: ignore +from google import auth # type: ignore from google.auth import credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore @@ -112,6 +114,7 @@ def __init__( channel: aio.Channel = None, api_mtls_endpoint: str = None, client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, + ssl_channel_credentials: grpc.ChannelCredentials = None, quota_project_id=None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, ) -> None: @@ -133,14 +136,16 @@ def __init__( are passed to :func:`google.auth.default`. channel (Optional[aio.Channel]): A ``Channel`` instance through which to make calls. - api_mtls_endpoint (Optional[str]): The mutual TLS endpoint. If - provided, it overrides the ``host`` argument and tries to create + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create a mutual TLS channel with client SSL credentials from ``client_cert_source`` or applicatin default SSL credentials. - client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): A - callback to provide client SSL certificate bytes and private key - bytes, both in PEM format. It is ignored if ``api_mtls_endpoint`` - is None. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for grpc channel. It is ignored if ``channel`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): @@ -163,12 +168,22 @@ def __init__( # If a channel was explicitly provided, set it. self._grpc_channel = channel elif api_mtls_endpoint: + warnings.warn( + "api_mtls_endpoint and client_cert_source are deprecated", + DeprecationWarning, + ) + host = ( api_mtls_endpoint if ":" in api_mtls_endpoint else api_mtls_endpoint + ":443" ) + if credentials is None: + credentials, _ = auth.default( + scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id + ) + # Create SSL credentials with client_cert_source or application # default SSL credentials. if client_cert_source: @@ -188,6 +203,23 @@ def __init__( scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, ) + else: + host = host if ":" in host else host + ":443" + + if credentials is None: + credentials, _ = auth.default( + scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id + ) + + # create a new channel. The provided one is ignored. + self._grpc_channel = type(self).create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + ssl_credentials=ssl_channel_credentials, + scopes=scopes or self.AUTH_SCOPES, + quota_project_id=quota_project_id, + ) # Run the base constructor. super().__init__( @@ -208,13 +240,6 @@ def grpc_channel(self) -> aio.Channel: This property caches on the instance; repeated calls return the same channel. """ - # Sanity check: Only create a new channel if we do not already - # have one. - if not hasattr(self, "_grpc_channel"): - self._grpc_channel = self.create_channel( - self._host, credentials=self._credentials, - ) - # Return the channel from cache. return self._grpc_channel diff --git a/packages/google-cloud-kms/noxfile.py b/packages/google-cloud-kms/noxfile.py index 49b02f0fd0fc..4cec8f3145bd 100644 --- a/packages/google-cloud-kms/noxfile.py +++ b/packages/google-cloud-kms/noxfile.py @@ -173,7 +173,9 @@ def docfx(session): """Build the docfx yaml files for this library.""" session.install("-e", ".") - session.install("sphinx", "alabaster", "recommonmark", "sphinx-docfx-yaml") + # sphinx-docfx-yaml supports up to sphinx version 1.5.5. + # https://github.com/docascode/sphinx-docfx-yaml/issues/97 + session.install("sphinx==1.5.5", "alabaster", "recommonmark", "sphinx-docfx-yaml") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( diff --git a/packages/google-cloud-kms/samples/snippets/README.md b/packages/google-cloud-kms/samples/snippets/README.md index bd4f0609f4cb..56715580b612 100644 --- a/packages/google-cloud-kms/samples/snippets/README.md +++ b/packages/google-cloud-kms/samples/snippets/README.md @@ -31,6 +31,7 @@ To run this sample: python quickstart.py + More information about the Cloud KMS quickstart is available at https://cloud.google.com/kms/docs/quickstart ## Additional Information diff --git a/packages/google-cloud-kms/scripts/decrypt-secrets.sh b/packages/google-cloud-kms/scripts/decrypt-secrets.sh index ff599eb2af25..21f6d2a26d90 100755 --- a/packages/google-cloud-kms/scripts/decrypt-secrets.sh +++ b/packages/google-cloud-kms/scripts/decrypt-secrets.sh @@ -20,14 +20,27 @@ ROOT=$( dirname "$DIR" ) # Work from the project root. cd $ROOT +# Prevent it from overriding files. +# We recommend that sample authors use their own service account files and cloud project. +# In that case, they are supposed to prepare these files by themselves. +if [[ -f "testing/test-env.sh" ]] || \ + [[ -f "testing/service-account.json" ]] || \ + [[ -f "testing/client-secrets.json" ]]; then + echo "One or more target files exist, aborting." + exit 1 +fi + # Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. PROJECT_ID="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" gcloud secrets versions access latest --secret="python-docs-samples-test-env" \ + --project="${PROJECT_ID}" \ > testing/test-env.sh gcloud secrets versions access latest \ --secret="python-docs-samples-service-account" \ + --project="${PROJECT_ID}" \ > testing/service-account.json gcloud secrets versions access latest \ --secret="python-docs-samples-client-secrets" \ - > testing/client-secrets.json \ No newline at end of file + --project="${PROJECT_ID}" \ + > testing/client-secrets.json diff --git a/packages/google-cloud-kms/setup.py b/packages/google-cloud-kms/setup.py index 4c3d2e4a47c4..563b789bfc67 100644 --- a/packages/google-cloud-kms/setup.py +++ b/packages/google-cloud-kms/setup.py @@ -24,9 +24,9 @@ version = "2.1.0" release_status = "Development Status :: 5 - Production/Stable" dependencies = [ - "google-api-core[grpc] >= 1.22.0, < 2.0.0dev", + "google-api-core[grpc] >= 1.22.2, < 2.0.0dev", "grpc-google-iam-v1 >= 0.12.3, < 0.13dev", - "proto-plus >= 0.4.0", + "proto-plus >= 1.4.0", "libcst >= 0.2.5", ] diff --git a/packages/google-cloud-kms/synth.metadata b/packages/google-cloud-kms/synth.metadata index 6ef33a598ac6..0387636aee51 100644 --- a/packages/google-cloud-kms/synth.metadata +++ b/packages/google-cloud-kms/synth.metadata @@ -4,29 +4,21 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-kms.git", - "sha": "9edcc9abceb6f790f4d1968aef3ba61903e30824" - } - }, - { - "git": { - "name": "googleapis", - "remote": "https://github.com/googleapis/googleapis.git", - "sha": "1a85976e6bcecdaee1242f6e7fd8b3db81904f56", - "internalRef": "328476173" + "sha": "81db5d90112092772b83aec57e2358088ed88e0d" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "2e85c10b5153defd9d654c34b57e7e9263361959" + "sha": "dba48bb9bc6959c232bec9150ac6313b608fe7bd" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "2e85c10b5153defd9d654c34b57e7e9263361959" + "sha": "dba48bb9bc6959c232bec9150ac6313b608fe7bd" } } ], diff --git a/packages/google-cloud-kms/tests/unit/gapic/kms_v1/test_key_management_service.py b/packages/google-cloud-kms/tests/unit/gapic/kms_v1/test_key_management_service.py index 8bda1d3a878a..f4598d4cc741 100644 --- a/packages/google-cloud-kms/tests/unit/gapic/kms_v1/test_key_management_service.py +++ b/packages/google-cloud-kms/tests/unit/gapic/kms_v1/test_key_management_service.py @@ -172,15 +172,14 @@ def test_key_management_service_client_client_options( credentials_file=None, host="squid.clam.whelk", scopes=None, - api_mtls_endpoint="squid.clam.whelk", - client_cert_source=None, + ssl_channel_credentials=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, ) - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS is + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is # "never". - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS": "never"}): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None client = client_class() @@ -189,15 +188,14 @@ def test_key_management_service_client_client_options( credentials_file=None, host=client.DEFAULT_ENDPOINT, scopes=None, - api_mtls_endpoint=client.DEFAULT_ENDPOINT, - client_cert_source=None, + ssl_channel_credentials=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, ) - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS is + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is # "always". - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS": "always"}): + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None client = client_class() @@ -206,95 +204,185 @@ def test_key_management_service_client_client_options( credentials_file=None, host=client.DEFAULT_MTLS_ENDPOINT, scopes=None, - api_mtls_endpoint=client.DEFAULT_MTLS_ENDPOINT, - client_cert_source=None, + ssl_channel_credentials=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, ) - # Check the case api_endpoint is not provided, GOOGLE_API_USE_MTLS is - # "auto", and client_cert_source is provided. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS": "auto"}): + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError): + client = client_class() + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + with pytest.raises(ValueError): + client = client_class() + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + ssl_channel_credentials=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + ) + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name,use_client_cert_env", + [ + ( + KeyManagementServiceClient, + transports.KeyManagementServiceGrpcTransport, + "grpc", + "true", + ), + ( + KeyManagementServiceAsyncClient, + transports.KeyManagementServiceGrpcAsyncIOTransport, + "grpc_asyncio", + "true", + ), + ( + KeyManagementServiceClient, + transports.KeyManagementServiceGrpcTransport, + "grpc", + "false", + ), + ( + KeyManagementServiceAsyncClient, + transports.KeyManagementServiceGrpcAsyncIOTransport, + "grpc_asyncio", + "false", + ), + ], +) +@mock.patch.object( + KeyManagementServiceClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(KeyManagementServiceClient), +) +@mock.patch.object( + KeyManagementServiceAsyncClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(KeyManagementServiceAsyncClient), +) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_key_management_service_client_mtls_env_auto( + client_class, transport_class, transport_name, use_client_cert_env +): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): options = client_options.ClientOptions( client_cert_source=client_cert_source_callback ) with mock.patch.object(transport_class, "__init__") as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_MTLS_ENDPOINT, - scopes=None, - api_mtls_endpoint=client.DEFAULT_MTLS_ENDPOINT, - client_cert_source=client_cert_source_callback, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - ) - - # Check the case api_endpoint is not provided, GOOGLE_API_USE_MTLS is - # "auto", and default_client_cert_source is provided. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS": "auto"}): - with mock.patch.object(transport_class, "__init__") as patched: + ssl_channel_creds = mock.Mock() with mock.patch( - "google.auth.transport.mtls.has_default_client_cert_source", - return_value=True, + "grpc.ssl_channel_credentials", return_value=ssl_channel_creds ): patched.return_value = None - client = client_class() + client = client_class(client_options=options) + + if use_client_cert_env == "false": + expected_ssl_channel_creds = None + expected_host = client.DEFAULT_ENDPOINT + else: + expected_ssl_channel_creds = ssl_channel_creds + expected_host = client.DEFAULT_MTLS_ENDPOINT + patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_MTLS_ENDPOINT, + host=expected_host, scopes=None, - api_mtls_endpoint=client.DEFAULT_MTLS_ENDPOINT, - client_cert_source=None, + ssl_channel_credentials=expected_ssl_channel_creds, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, ) - # Check the case api_endpoint is not provided, GOOGLE_API_USE_MTLS is - # "auto", but client_cert_source and default_client_cert_source are None. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS": "auto"}): + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): with mock.patch.object(transport_class, "__init__") as patched: with mock.patch( - "google.auth.transport.mtls.has_default_client_cert_source", - return_value=False, + "google.auth.transport.grpc.SslCredentials.__init__", return_value=None ): - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=None, - api_mtls_endpoint=client.DEFAULT_ENDPOINT, - client_cert_source=None, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - ) - - # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS has - # unsupported value. - with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS": "Unsupported"}): - with pytest.raises(MutualTLSChannelError): - client = client_class() - - # Check the case quota_project_id is provided - options = client_options.ClientOptions(quota_project_id="octopus") - with mock.patch.object(transport_class, "__init__") as patched: - patched.return_value = None - client = client_class(client_options=options) - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=client.DEFAULT_ENDPOINT, - scopes=None, - api_mtls_endpoint=client.DEFAULT_ENDPOINT, - client_cert_source=None, - quota_project_id="octopus", - client_info=transports.base.DEFAULT_CLIENT_INFO, - ) + with mock.patch( + "google.auth.transport.grpc.SslCredentials.is_mtls", + new_callable=mock.PropertyMock, + ) as is_mtls_mock: + with mock.patch( + "google.auth.transport.grpc.SslCredentials.ssl_credentials", + new_callable=mock.PropertyMock, + ) as ssl_credentials_mock: + if use_client_cert_env == "false": + is_mtls_mock.return_value = False + ssl_credentials_mock.return_value = None + expected_host = client.DEFAULT_ENDPOINT + expected_ssl_channel_creds = None + else: + is_mtls_mock.return_value = True + ssl_credentials_mock.return_value = mock.Mock() + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_ssl_channel_creds = ( + ssl_credentials_mock.return_value + ) + + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + ssl_channel_credentials=expected_ssl_channel_creds, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + with mock.patch.object(transport_class, "__init__") as patched: + with mock.patch( + "google.auth.transport.grpc.SslCredentials.__init__", return_value=None + ): + with mock.patch( + "google.auth.transport.grpc.SslCredentials.is_mtls", + new_callable=mock.PropertyMock, + ) as is_mtls_mock: + is_mtls_mock.return_value = False + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + ssl_channel_credentials=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + ) @pytest.mark.parametrize( @@ -325,8 +413,7 @@ def test_key_management_service_client_client_options_scopes( credentials_file=None, host=client.DEFAULT_ENDPOINT, scopes=["1", "2"], - api_mtls_endpoint=client.DEFAULT_ENDPOINT, - client_cert_source=None, + ssl_channel_credentials=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, ) @@ -360,8 +447,7 @@ def test_key_management_service_client_client_options_credentials_file( credentials_file="credentials.json", host=client.DEFAULT_ENDPOINT, scopes=None, - api_mtls_endpoint=client.DEFAULT_ENDPOINT, - client_cert_source=None, + ssl_channel_credentials=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, ) @@ -380,8 +466,7 @@ def test_key_management_service_client_client_options_from_dict(): credentials_file=None, host="squid.clam.whelk", scopes=None, - api_mtls_endpoint="squid.clam.whelk", - client_cert_source=None, + ssl_channel_credentials=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, ) @@ -6151,6 +6236,21 @@ def test_transport_get_channel(): assert channel +@pytest.mark.parametrize( + "transport_class", + [ + transports.KeyManagementServiceGrpcTransport, + transports.KeyManagementServiceGrpcAsyncIOTransport, + ], +) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(auth, "default") as adc: + adc.return_value = (credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + + def test_transport_grpc_default(): # A client should use the gRPC transport by default. client = KeyManagementServiceClient(credentials=credentials.AnonymousCredentials(),) @@ -6233,6 +6333,17 @@ def test_key_management_service_base_transport_with_credentials_file(): ) +def test_key_management_service_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(auth, "default") as adc, mock.patch( + "google.cloud.kms_v1.services.key_management_service.transports.KeyManagementServiceTransport._prep_wrapped_messages" + ) as Transport: + Transport.return_value = None + adc.return_value = (credentials.AnonymousCredentials(), None) + transport = transports.KeyManagementServiceTransport() + adc.assert_called_once() + + def test_key_management_service_auth_adc(): # If no credentials are provided, we should use ADC credentials. with mock.patch.object(auth, "default") as adc: @@ -6287,191 +6398,116 @@ def test_key_management_service_host_with_port(): def test_key_management_service_grpc_transport_channel(): channel = grpc.insecure_channel("http://localhost/") - # Check that if channel is provided, mtls endpoint and client_cert_source - # won't be used. - callback = mock.MagicMock() + # Check that channel is used if provided. transport = transports.KeyManagementServiceGrpcTransport( - host="squid.clam.whelk", - channel=channel, - api_mtls_endpoint="mtls.squid.clam.whelk", - client_cert_source=callback, + host="squid.clam.whelk", channel=channel, ) assert transport.grpc_channel == channel assert transport._host == "squid.clam.whelk:443" - assert not callback.called def test_key_management_service_grpc_asyncio_transport_channel(): channel = aio.insecure_channel("http://localhost/") - # Check that if channel is provided, mtls endpoint and client_cert_source - # won't be used. - callback = mock.MagicMock() + # Check that channel is used if provided. transport = transports.KeyManagementServiceGrpcAsyncIOTransport( - host="squid.clam.whelk", - channel=channel, - api_mtls_endpoint="mtls.squid.clam.whelk", - client_cert_source=callback, + host="squid.clam.whelk", channel=channel, ) assert transport.grpc_channel == channel assert transport._host == "squid.clam.whelk:443" - assert not callback.called - - -@mock.patch("grpc.ssl_channel_credentials", autospec=True) -@mock.patch("google.api_core.grpc_helpers.create_channel", autospec=True) -def test_key_management_service_grpc_transport_channel_mtls_with_client_cert_source( - grpc_create_channel, grpc_ssl_channel_cred -): - # Check that if channel is None, but api_mtls_endpoint and client_cert_source - # are provided, then a mTLS channel will be created. - mock_cred = mock.Mock() - - mock_ssl_cred = mock.Mock() - grpc_ssl_channel_cred.return_value = mock_ssl_cred - - mock_grpc_channel = mock.Mock() - grpc_create_channel.return_value = mock_grpc_channel - - transport = transports.KeyManagementServiceGrpcTransport( - host="squid.clam.whelk", - credentials=mock_cred, - api_mtls_endpoint="mtls.squid.clam.whelk", - client_cert_source=client_cert_source_callback, - ) - grpc_ssl_channel_cred.assert_called_once_with( - certificate_chain=b"cert bytes", private_key=b"key bytes" - ) - grpc_create_channel.assert_called_once_with( - "mtls.squid.clam.whelk:443", - credentials=mock_cred, - credentials_file=None, - scopes=( - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/cloudkms", - ), - ssl_credentials=mock_ssl_cred, - quota_project_id=None, - ) - assert transport.grpc_channel == mock_grpc_channel - - -@mock.patch("grpc.ssl_channel_credentials", autospec=True) -@mock.patch("google.api_core.grpc_helpers_async.create_channel", autospec=True) -def test_key_management_service_grpc_asyncio_transport_channel_mtls_with_client_cert_source( - grpc_create_channel, grpc_ssl_channel_cred -): - # Check that if channel is None, but api_mtls_endpoint and client_cert_source - # are provided, then a mTLS channel will be created. - mock_cred = mock.Mock() - - mock_ssl_cred = mock.Mock() - grpc_ssl_channel_cred.return_value = mock_ssl_cred - - mock_grpc_channel = mock.Mock() - grpc_create_channel.return_value = mock_grpc_channel - - transport = transports.KeyManagementServiceGrpcAsyncIOTransport( - host="squid.clam.whelk", - credentials=mock_cred, - api_mtls_endpoint="mtls.squid.clam.whelk", - client_cert_source=client_cert_source_callback, - ) - grpc_ssl_channel_cred.assert_called_once_with( - certificate_chain=b"cert bytes", private_key=b"key bytes" - ) - grpc_create_channel.assert_called_once_with( - "mtls.squid.clam.whelk:443", - credentials=mock_cred, - credentials_file=None, - scopes=( - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/cloudkms", - ), - ssl_credentials=mock_ssl_cred, - quota_project_id=None, - ) - assert transport.grpc_channel == mock_grpc_channel @pytest.mark.parametrize( - "api_mtls_endpoint", ["mtls.squid.clam.whelk", "mtls.squid.clam.whelk:443"] + "transport_class", + [ + transports.KeyManagementServiceGrpcTransport, + transports.KeyManagementServiceGrpcAsyncIOTransport, + ], ) -@mock.patch("google.api_core.grpc_helpers.create_channel", autospec=True) -def test_key_management_service_grpc_transport_channel_mtls_with_adc( - grpc_create_channel, api_mtls_endpoint +def test_key_management_service_transport_channel_mtls_with_client_cert_source( + transport_class, ): - # Check that if channel and client_cert_source are None, but api_mtls_endpoint - # is provided, then a mTLS channel will be created with SSL ADC. - mock_grpc_channel = mock.Mock() - grpc_create_channel.return_value = mock_grpc_channel - - # Mock google.auth.transport.grpc.SslCredentials class. - mock_ssl_cred = mock.Mock() - with mock.patch.multiple( - "google.auth.transport.grpc.SslCredentials", - __init__=mock.Mock(return_value=None), - ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), - ): - mock_cred = mock.Mock() - transport = transports.KeyManagementServiceGrpcTransport( - host="squid.clam.whelk", - credentials=mock_cred, - api_mtls_endpoint=api_mtls_endpoint, - client_cert_source=None, - ) - grpc_create_channel.assert_called_once_with( - "mtls.squid.clam.whelk:443", - credentials=mock_cred, - credentials_file=None, - scopes=( - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/cloudkms", - ), - ssl_credentials=mock_ssl_cred, - quota_project_id=None, - ) - assert transport.grpc_channel == mock_grpc_channel + with mock.patch( + "grpc.ssl_channel_credentials", autospec=True + ) as grpc_ssl_channel_cred: + with mock.patch.object( + transport_class, "create_channel", autospec=True + ) as grpc_create_channel: + mock_ssl_cred = mock.Mock() + grpc_ssl_channel_cred.return_value = mock_ssl_cred + + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + + cred = credentials.AnonymousCredentials() + with pytest.warns(DeprecationWarning): + with mock.patch.object(auth, "default") as adc: + adc.return_value = (cred, None) + transport = transport_class( + host="squid.clam.whelk", + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=client_cert_source_callback, + ) + adc.assert_called_once() + + grpc_ssl_channel_cred.assert_called_once_with( + certificate_chain=b"cert bytes", private_key=b"key bytes" + ) + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=( + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloudkms", + ), + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + ) + assert transport.grpc_channel == mock_grpc_channel @pytest.mark.parametrize( - "api_mtls_endpoint", ["mtls.squid.clam.whelk", "mtls.squid.clam.whelk:443"] + "transport_class", + [ + transports.KeyManagementServiceGrpcTransport, + transports.KeyManagementServiceGrpcAsyncIOTransport, + ], ) -@mock.patch("google.api_core.grpc_helpers_async.create_channel", autospec=True) -def test_key_management_service_grpc_asyncio_transport_channel_mtls_with_adc( - grpc_create_channel, api_mtls_endpoint -): - # Check that if channel and client_cert_source are None, but api_mtls_endpoint - # is provided, then a mTLS channel will be created with SSL ADC. - mock_grpc_channel = mock.Mock() - grpc_create_channel.return_value = mock_grpc_channel - - # Mock google.auth.transport.grpc.SslCredentials class. +def test_key_management_service_transport_channel_mtls_with_adc(transport_class): mock_ssl_cred = mock.Mock() with mock.patch.multiple( "google.auth.transport.grpc.SslCredentials", __init__=mock.Mock(return_value=None), ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), ): - mock_cred = mock.Mock() - transport = transports.KeyManagementServiceGrpcAsyncIOTransport( - host="squid.clam.whelk", - credentials=mock_cred, - api_mtls_endpoint=api_mtls_endpoint, - client_cert_source=None, - ) - grpc_create_channel.assert_called_once_with( - "mtls.squid.clam.whelk:443", - credentials=mock_cred, - credentials_file=None, - scopes=( - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/cloudkms", - ), - ssl_credentials=mock_ssl_cred, - quota_project_id=None, - ) - assert transport.grpc_channel == mock_grpc_channel + with mock.patch.object( + transport_class, "create_channel", autospec=True + ) as grpc_create_channel: + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + mock_cred = mock.Mock() + + with pytest.warns(DeprecationWarning): + transport = transport_class( + host="squid.clam.whelk", + credentials=mock_cred, + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=None, + ) + + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=mock_cred, + credentials_file=None, + scopes=( + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloudkms", + ), + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + ) + assert transport.grpc_channel == mock_grpc_channel def test_crypto_key_path(): @@ -6503,35 +6539,6 @@ def test_parse_crypto_key_path(): assert expected == actual -def test_import_job_path(): - project = "squid" - location = "clam" - key_ring = "whelk" - import_job = "octopus" - - expected = "projects/{project}/locations/{location}/keyRings/{key_ring}/importJobs/{import_job}".format( - project=project, location=location, key_ring=key_ring, import_job=import_job, - ) - actual = KeyManagementServiceClient.import_job_path( - project, location, key_ring, import_job - ) - assert expected == actual - - -def test_parse_import_job_path(): - expected = { - "project": "oyster", - "location": "nudibranch", - "key_ring": "cuttlefish", - "import_job": "mussel", - } - path = KeyManagementServiceClient.import_job_path(**expected) - - # Check that the path construction is reversible. - actual = KeyManagementServiceClient.parse_import_job_path(path) - assert expected == actual - - def test_crypto_key_version_path(): project = "squid" location = "clam" @@ -6567,6 +6574,35 @@ def test_parse_crypto_key_version_path(): assert expected == actual +def test_import_job_path(): + project = "squid" + location = "clam" + key_ring = "whelk" + import_job = "octopus" + + expected = "projects/{project}/locations/{location}/keyRings/{key_ring}/importJobs/{import_job}".format( + project=project, location=location, key_ring=key_ring, import_job=import_job, + ) + actual = KeyManagementServiceClient.import_job_path( + project, location, key_ring, import_job + ) + assert expected == actual + + +def test_parse_import_job_path(): + expected = { + "project": "oyster", + "location": "nudibranch", + "key_ring": "cuttlefish", + "import_job": "mussel", + } + path = KeyManagementServiceClient.import_job_path(**expected) + + # Check that the path construction is reversible. + actual = KeyManagementServiceClient.parse_import_job_path(path) + assert expected == actual + + def test_key_ring_path(): project = "squid" location = "clam"