cloudiot_mqtt_example_test: test_gateway_send_data_for_device failed

[flaky-bot]

Note: #3912 was also for this test, but it was closed more than 10 days ago. So, I didn't mark it flaky.

---

commit: 02abb31
buildURL: Build Status, Sponge
status: failed

Test output

args = (parent: "projects/python-docs-samples-tests/locations/us-central1/registries/test-registry-e62b1af7e2fe4b4daaeaa7efb642ebfe-1607948911"
,)
kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/python-docs-samples-tests/locations/us-central1/registries/te...af7e2fe4b4daaeaa7efb642ebfe-1607948911'), ('x-goog-api-client', 'gl-python/3.8.3 grpc/1.34.0 gax/1.23.0 gapic/2.0.1')]}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/py-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:57:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7ff2c6e06610>

request = parent: "projects/python-docs-samples-tests/locations/us-central1/registries/test-registry-e62b1af7e2fe4b4daaeaa7efb642ebfe-1607948911"
timeout = None

metadata = [('x-goog-request-params', 'parent=projects/python-docs-samples-tests/locations/us-central1/registries/test-registry-e62b1af7e2fe4b4daaeaa7efb642ebfe-1607948911'), ('x-goog-api-client', 'gl-python/3.8.3 grpc/1.34.0 gax/1.23.0 gapic/2.0.1')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/py-3-8/lib/python3.8/site-packages/grpc/_channel.py:923:

state = <grpc._channel._RPCState object at 0x7ff2c707c5e0>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7ff2c716f6c0>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.UNAVAILABLE

E           	details = "Getting metadata from plugin failed with error: Expecting value: line 1 column 1 (char 0)"

E           	debug_error_string = "{"created":"@1607949259.101350905","description":"Getting metadata from plugin failed with error: Expecting value: line 1 column 1 (char 0)","file":"src/core/lib/security/credentials/plugin/plugin_credentials.cc","file_line":90,"grpc_status":14}"

E           >
.nox/py-3-8/lib/python3.8/site-packages/grpc/_channel.py:826: _InactiveRpcError
The above exception was the direct cause of the following exception:
target = functools.partial(<function _wrap_unary_errors..error_remapped_callable at 0x7ff2d4096f70>, parent: "projects/...af7e2fe4b4daaeaa7efb642ebfe-1607948911'), ('x-goog-api-client', 'gl-python/3.8.3 grpc/1.34.0 gax/1.23.0 gapic/2.0.1')])

predicate = <function if_exception_type..if_exception_type_predicate at 0x7ff2d4096ee0>

sleep_generator = <generator object exponential_sleep_generator at 0x7ff2c70cedd0>

deadline = 120.0, on_error = None
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:


          return target()


.nox/py-3-8/lib/python3.8/site-packages/google/api_core/retry.py:184:

args = (parent: "projects/python-docs-samples-tests/locations/us-central1/registries/test-registry-e62b1af7e2fe4b4daaeaa7efb642ebfe-1607948911"

,)

kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/python-docs-samples-tests/locations/us-central1/registries/te...af7e2fe4b4daaeaa7efb642ebfe-1607948911'), ('x-goog-api-client', 'gl-python/3.8.3 grpc/1.34.0 gax/1.23.0 gapic/2.0.1')]}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:
        return callable_(*args, **kwargs)
    except grpc.RpcError as exc:


      six.raise_from(exceptions.from_grpc_error(exc), exc)


.nox/py-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:59:

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.UNAVAILABLE

details = "Getting metadata from plu...1 (char 0)","file":"src/core/lib/security/credentials/plugin/plugin_credentials.cc","file_line":90,"grpc_status":14}"



???

E   google.api_core.exceptions.ServiceUnavailable: 503 Getting metadata from plugin failed with error: Expecting value: line 1 column 1 (char 0)

:3: ServiceUnavailable
The above exception was the direct cause of the following exception:
test_registry_id = 'test-registry-e62b1af7e2fe4b4daaeaa7efb642ebfe-1607948911'
@pytest.fixture(scope='module')
def device_and_gateways(test_registry_id):
    device_id = device_id_template.format('noauthbind')
    gateway_id = device_id_template.format('RS256')
    bad_gateway_id = device_id_template.format('RS256-err')

    @backoff.on_exception(backoff.expo, HttpError, max_time=60)
    def create_device():
        manager.create_device(
            service_account_json, project_id, cloud_region, test_registry_id,
            device_id)


  create_device()


    @backoff.on_exception(backoff.expo, HttpError, max_time=60)

fixtures.py:152:

.nox/py-3-8/lib/python3.8/site-packages/backoff/_sync.py:94: in retry

ret = target(*args, **kwargs)

fixtures.py:149: in create_device

manager.create_device(

../manager/manager.py:159: in create_device

devices = list(client.list_devices(request={"parent": parent}))

.nox/py-3-8/lib/python3.8/site-packages/google/cloud/iot_v1/services/device_manager/client.py:1029: in list_devices

response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/py-3-8/lib/python3.8/site-packages/google/api_core/retry.py:281: in retry_wrapped_func

return retry_target(

target = functools.partial(<function _wrap_unary_errors..error_remapped_callable at 0x7ff2d4096f70>, parent: "projects/...af7e2fe4b4daaeaa7efb642ebfe-1607948911'), ('x-goog-api-client', 'gl-python/3.8.3 grpc/1.34.0 gax/1.23.0 gapic/2.0.1')])

predicate = <function if_exception_type..if_exception_type_predicate at 0x7ff2d4096ee0>

sleep_generator = <generator object exponential_sleep_generator at 0x7ff2c70cedd0>

deadline = 120.0, on_error = None
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:
            return target()

        # pylint: disable=broad-except
        # This function explicitly must deal with broad exceptions.
        except Exception as exc:
            if not predicate(exc):
                raise
            last_exc = exc
            if on_error is not None:
                on_error(exc)

        now = datetime_helpers.utcnow()

        if deadline_datetime is not None:
            if deadline_datetime <= now:


              six.raise_from(


                    exceptions.RetryError(
                        "Deadline of {:.1f}s exceeded while calling {}".format(
                            deadline, target
                        ),
                        last_exc,
                    ),
                    last_exc,
                )

E                   google.api_core.exceptions.RetryError: Deadline of 120.0s exceeded while calling functools.partial(<function _wrap_unary_errors..error_remapped_callable at 0x7ff2d4096f70>, parent: "projects/python-docs-samples-tests/locations/us-central1/registries/test-registry-e62b1af7e2fe4b4daaeaa7efb642ebfe-1607948911"

E                   , metadata=[('x-goog-request-params', 'parent=projects/python-docs-samples-tests/locations/us-central1/registries/test-registry-e62b1af7e2fe4b4daaeaa7efb642ebfe-1607948911'), ('x-goog-api-client', 'gl-python/3.8.3 grpc/1.34.0 gax/1.23.0 gapic/2.0.1')]), last exception: 503 Getting metadata from plugin failed with error: Expecting value: line 1 column 1 (char 0)
.nox/py-3-8/lib/python3.8/site-packages/google/api_core/retry.py:199: RetryError

[busunkim96]

Error appears to be transient, closing 503 Getting metadata from plugin failed with error: Expecting value: line 1 column 1 (char 0)