Sample test test_track_objects_streaming failing with "Stream removed"

[parthea]

The sample test test_track_objects_streaming has started failing with google.api_core.exceptions.Unknown: None Stream removed

Stack Trace

=================================== FAILURES ===================================
_________________________ test_track_objects_streaming _________________________

self = <google.api_core.grpc_helpers._StreamingResponseIterator object at 0x7f574c0e54d0>

    def __next__(self):
        """Get the next response from the stream.

        Returns:
            protobuf.Message: A single response from the stream.
        """
        try:
            if hasattr(self, "_stored_first_result"):
                result = self._stored_first_result
                del self._stored_first_result
                return result
>           return next(self._wrapped)

.nox/py-3-7/lib/python3.7/site-packages/google/api_core/grpc_helpers.py:97:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <_MultiThreadedRendezvous of RPC that terminated with:
	status = StatusCode.UNKNOWN
	details = "Stream removed"
	debug...<REDACTED>:443","file":"src/core/lib/surface/call.cc","file_line":952,"grpc_message":"Stream removed","grpc_status":2}"
>

    def __next__(self):
>       return self._next()

.nox/py-3-7/lib/python3.7/site-packages/grpc/_channel.py:426:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <_MultiThreadedRendezvous of RPC that terminated with:
	status = StatusCode.UNKNOWN
	details = "Stream removed"
	debug...<REDACTED>:443","file":"src/core/lib/surface/call.cc","file_line":952,"grpc_message":"Stream removed","grpc_status":2}"
>

    def _next(self):
        with self._state.condition:
            if self._state.code is None:
                event_handler = _event_handler(self._state,
                                               self._response_deserializer)
                self._state.due.add(cygrpc.OperationType.receive_message)
                operating = self._call.operate(
                    (cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS),),
                    event_handler)
                if not operating:
                    self._state.due.remove(cygrpc.OperationType.receive_message)
            elif self._state.code is grpc.StatusCode.OK:
                raise StopIteration()
            else:
                raise self

            def _response_ready():
                return (self._state.response is not None or
                        (cygrpc.OperationType.receive_message
                         not in self._state.due and
                         self._state.code is not None))

            _common.wait(self._state.condition.wait, _response_ready)
            if self._state.response is not None:
                response = self._state.response
                self._state.response = None
                return response
            elif cygrpc.OperationType.receive_message not in self._state.due:
                if self._state.code is grpc.StatusCode.OK:
                    raise StopIteration()
                elif self._state.code is not None:
>                   raise self
E                   grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
E                   	status = StatusCode.UNKNOWN
E                   	details = "Stream removed"
E                   	debug_error_string = "{"created":"@1651769761.524382572","description":"Error received from peer ipv4:<REDACTED>:443","file":"src/core/lib/surface/call.cc","file_line":952,"grpc_message":"Stream removed","grpc_status":2}"
E                   >

.nox/py-3-7/lib/python3.7/site-packages/grpc/_channel.py:826: _MultiThreadedRendezvous

The above exception was the direct cause of the following exception:

capsys = <_pytest.capture.CaptureFixture object at 0x7f574c39f0d0>
video_path = '/tmp/pytest-of-kbuilder/pytest-0/video0/file.mp4'

    @pytest.mark.slow
    @pytest.mark.flaky(max_runs=3, min_passes=1)
    def test_track_objects_streaming(capsys, video_path):
>       beta_snippets.track_objects_streaming(video_path)

beta_snippets_test.py:102:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
beta_snippets.py:396: in track_objects_streaming
    for response in responses:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <google.api_core.grpc_helpers._StreamingResponseIterator object at 0x7f574c0e54d0>

    def __next__(self):
        """Get the next response from the stream.

        Returns:
            protobuf.Message: A single response from the stream.
        """
        try:
            if hasattr(self, "_stored_first_result"):
                result = self._stored_first_result
                del self._stored_first_result
                return result
            return next(self._wrapped)
        except grpc.RpcError as exc:
            # If the stream has already returned data, we cannot recover here.
>           raise exceptions.from_grpc_error(exc) from exc
E           google.api_core.exceptions.Unknown: None Stream removed

See #322 and build log here.

[galz10]

Hi @parthea i created a bug for this issue, i believe it has to do with something with the backend b/232260723

[meredithslota]

I marked this issue blocked by the internal issue linked above.

[parthea]

The current error is The Model does not exist. See build log here.

___________________ test_streaming_automl_action_recognition ___________________

capsys = <_pytest.capture.CaptureFixture object at 0x7fb63ce65f90>
video_path = '/tmp/pytest-of-kbuilder/pytest-0/video0/file.mp4'

    @pytest.mark.flaky(max_runs=3, min_passes=1)
    def test_streaming_automl_action_recognition(capsys, video_path):
        project_id = os.environ["GOOGLE_CLOUD_PROJECT"]
        model_id = "6597443600609443840"
        beta_snippets.streaming_automl_action_recognition(video_path, project_id, model_id)
        out, _ = capsys.readouterr()
>       assert "segment" in out
E       AssertionError: assert 'segment' in 'com.google.cloud.ai.platform.common.errors.AiPlatformException: code=NOT_FOUND, message=The Model does not exist., cause=null\n'