From e88b47ebcdd4036fe0db854d9ea316245f45b28f Mon Sep 17 00:00:00 2001 From: davidliu Date: Tue, 10 May 2022 21:31:59 +0900 Subject: [PATCH] Start/Stop receiving stream method for VideoTrack (#25) * Initial implementation of start/stop receive * move set_state call to signalling thread as well. https://source.chromium.org/chromium/_/webrtc/src.git/+/dfd69c22100e1d2e83295f33d06fa9916caa5c53 * separate should_receive from enabled for now * ios hooks * Fix ios compile errors * code cleanup * clean up * reuse frame buffers instead --- api/media_stream_interface.cc | 4 ++ api/media_stream_interface.h | 2 + media/base/media_channel.h | 3 ++ media/engine/webrtc_video_engine.cc | 29 +++++++++++++ media/engine/webrtc_video_engine.h | 6 ++- modules/video_coding/frame_buffer2.cc | 7 ++++ modules/video_coding/frame_buffer2.h | 2 + pc/media_stream_track_proxy.h | 2 + pc/video_rtp_receiver.cc | 43 +++++++++++++++++++- pc/video_rtp_receiver.h | 11 ++++- pc/video_track.cc | 14 +++++++ pc/video_track.h | 4 ++ sdk/android/api/org/webrtc/VideoTrack.java | 20 +++++++++ sdk/android/src/jni/video_track.cc | 11 +++++ sdk/objc/api/peerconnection/RTCVideoTrack.h | 3 ++ sdk/objc/api/peerconnection/RTCVideoTrack.mm | 8 ++++ video/video_receive_stream2.cc | 1 + 17 files changed, 167 insertions(+), 3 deletions(-) diff --git a/api/media_stream_interface.cc b/api/media_stream_interface.cc index e07907917b..5362522262 100644 --- a/api/media_stream_interface.cc +++ b/api/media_stream_interface.cc @@ -18,6 +18,10 @@ const char* const MediaStreamTrackInterface::kVideoKind = const char* const MediaStreamTrackInterface::kAudioKind = cricket::kMediaTypeAudio; +bool VideoTrackInterface::should_receive() const { + return true; +} + VideoTrackInterface::ContentHint VideoTrackInterface::content_hint() const { return ContentHint::kNone; } diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h index 9d336739e4..6d9b4aa6ca 100644 --- a/api/media_stream_interface.h +++ b/api/media_stream_interface.h @@ -188,6 +188,8 @@ class RTC_EXPORT VideoTrackInterface virtual VideoTrackSourceInterface* GetSource() const = 0; + virtual void set_should_receive(bool should_receive) {} + virtual bool should_receive() const; virtual ContentHint content_hint() const; virtual void set_content_hint(ContentHint hint) {} diff --git a/media/base/media_channel.h b/media/base/media_channel.h index ec78ec8c25..c62d4076ae 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -913,6 +913,9 @@ class VideoMediaChannel : public MediaChannel, public Delayable { virtual void GenerateKeyFrame(uint32_t ssrc) = 0; virtual std::vector GetSources(uint32_t ssrc) const = 0; + + virtual void StartReceive(uint32_t ssrc) {} + virtual void StopReceive(uint32_t ssrc) {} }; // Info about data received in DataMediaChannel. For use in diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 5b3ecae728..eb11d15169 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -970,6 +970,24 @@ void WebRtcVideoChannel::RequestEncoderSwitch( } } +void WebRtcVideoChannel::StartReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StartStream(); +} + +void WebRtcVideoChannel::StopReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StopStream(); +} + bool WebRtcVideoChannel::ApplyChangedParams( const ChangedSendParameters& changed_params) { RTC_DCHECK_RUN_ON(&thread_checker_); @@ -3056,6 +3074,17 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters( } } +void WebRtcVideoChannel::WebRtcVideoReceiveStream::StartStream(){ + if (stream_) { + stream_->Start(); + } +} +void WebRtcVideoChannel::WebRtcVideoReceiveStream::StopStream(){ + if (stream_) { + stream_->Stop(); + } +} + void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateReceiveStream() { absl::optional base_minimum_playout_delay_ms; absl::optional diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index b2780c4af5..6e7aa7b5f0 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -254,7 +254,8 @@ class WebRtcVideoChannel : public VideoMediaChannel, uint32_t ssrc, rtc::scoped_refptr frame_transformer) override; - + void StartReceive(uint32_t ssrc) override; + void StopReceive(uint32_t ssrc) override; private: class WebRtcVideoReceiveStream; @@ -501,6 +502,9 @@ class WebRtcVideoChannel : public VideoMediaChannel, void SetDepacketizerToDecoderFrameTransformer( rtc::scoped_refptr frame_transformer); + + void StartStream(); + void StopStream(); void SetLocalSsrc(uint32_t local_ssrc); diff --git a/modules/video_coding/frame_buffer2.cc b/modules/video_coding/frame_buffer2.cc index 7e6d622d5e..8901888c4f 100644 --- a/modules/video_coding/frame_buffer2.cc +++ b/modules/video_coding/frame_buffer2.cc @@ -332,6 +332,13 @@ void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { protection_mode_ = mode; } +void FrameBuffer::Start() { + TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); + MutexLock lock(&mutex_); + if (!stopped_) + return; + stopped_ = false; +} void FrameBuffer::Stop() { TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); MutexLock lock(&mutex_); diff --git a/modules/video_coding/frame_buffer2.h b/modules/video_coding/frame_buffer2.h index 1315d2f254..0d92835f29 100644 --- a/modules/video_coding/frame_buffer2.h +++ b/modules/video_coding/frame_buffer2.h @@ -78,6 +78,8 @@ class FrameBuffer { // Stop the frame buffer, causing any sleeping thread in NextFrame to // return immediately. void Stop(); + // Unstop the frame buffer, re-allowing new frames to be inserted and read. + void Start(); // Updates the RTT for jitter buffer estimation. void UpdateRtt(int64_t rtt_ms); diff --git a/pc/media_stream_track_proxy.h b/pc/media_stream_track_proxy.h index 2af3aedb22..fab23d17ec 100644 --- a/pc/media_stream_track_proxy.h +++ b/pc/media_stream_track_proxy.h @@ -55,6 +55,8 @@ PROXY_SECONDARY_METHOD2(void, PROXY_SECONDARY_METHOD1(void, RemoveSink, rtc::VideoSinkInterface*) PROXY_SECONDARY_METHOD0(void, RequestRefreshFrame) BYPASS_PROXY_CONSTMETHOD0(VideoTrackSourceInterface*, GetSource) +PROXY_CONSTMETHOD0(bool, should_receive) +PROXY_METHOD1(void, set_should_receive, bool) PROXY_METHOD1(void, RegisterObserver, ObserverInterface*) PROXY_METHOD1(void, UnregisterObserver, ObserverInterface*) diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index 3cc568f12a..ac912a7820 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -22,6 +22,7 @@ #include "rtc_base/location.h" #include "rtc_base/logging.h" #include "rtc_base/ref_counted_object.h" +#include "rtc_base/task_utils/to_queued_task.h" namespace webrtc { @@ -43,9 +44,12 @@ VideoRtpReceiver::VideoRtpReceiver( rtc::Thread::Current(), worker_thread, VideoTrack::Create(receiver_id, source_, worker_thread))), - attachment_id_(GenerateUniqueId()) { + cached_track_should_receive_(track_->should_receive()), + attachment_id_(GenerateUniqueId()), + worker_thread_safety_(PendingTaskSafetyFlag::CreateDetachedInactive()) { RTC_DCHECK(worker_thread_); SetStreams(streams); + track_->RegisterObserver(this); RTC_DCHECK_EQ(source_->state(), MediaSourceInterface::kInitializing); } @@ -114,6 +118,40 @@ void VideoRtpReceiver::Stop() { track_->internal()->set_ended(); } +void VideoRtpReceiver::OnChanged() { + RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + if (cached_track_should_receive_ != track_->should_receive()) { + cached_track_should_receive_ = track_->should_receive(); + worker_thread_->PostTask(ToQueuedTask( + worker_thread_safety_, + [this, receive = cached_track_should_receive_]() { + RTC_DCHECK_RUN_ON(worker_thread_); + if(receive) { + StartMediaChannel(); + } else { + StopMediaChannel(); + } + })); + } +} + +void VideoRtpReceiver::StartMediaChannel() { + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StartReceive(ssrc_.value_or(0)); + OnGenerateKeyFrame(); +} + +void VideoRtpReceiver::StopMediaChannel() { + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StopReceive(ssrc_.value_or(0)); +} + // RTC_RUN_ON(&signaling_thread_checker_) void VideoRtpReceiver::RestartMediaChannel(absl::optional ssrc) { MediaSourceInterface::SourceState state = source_->state(); @@ -205,6 +243,7 @@ void VideoRtpReceiver::set_transport( void VideoRtpReceiver::SetStreams( const std::vector>& streams) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + // Remove remote track from any streams that are going away. for (const auto& existing_stream : streams_) { bool removed = true; @@ -276,6 +315,8 @@ void VideoRtpReceiver::SetMediaChannel_w(cricket::MediaChannel* media_channel) { SetEncodedSinkEnabled(false); } + media_channel ? worker_thread_safety_->SetAlive() + : worker_thread_safety_->SetNotAlive(); media_channel_ = static_cast(media_channel); if (media_channel_) { diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index 05532a2eb1..229e648ee5 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -43,7 +43,8 @@ namespace webrtc { -class VideoRtpReceiver : public RtpReceiverInternal { +class VideoRtpReceiver : public RtpReceiverInternal, + public ObserverInterface { public: // An SSRC of 0 will create a receiver that will match the first SSRC it // sees. Must be called on signaling thread. @@ -61,6 +62,9 @@ class VideoRtpReceiver : public RtpReceiverInternal { rtc::scoped_refptr video_track() const { return track_; } + // ObserverInterface implementation + void OnChanged() override; + // RtpReceiverInterface implementation rtc::scoped_refptr track() const override { return track_; @@ -115,6 +119,8 @@ class VideoRtpReceiver : public RtpReceiverInternal { cricket::MediaChannel* media_channel); private: + void StartMediaChannel(); + void StopMediaChannel(); void RestartMediaChannel(absl::optional ssrc) RTC_RUN_ON(&signaling_thread_checker_); void RestartMediaChannel_w(absl::optional ssrc, @@ -162,6 +168,8 @@ class VideoRtpReceiver : public RtpReceiverInternal { RTC_GUARDED_BY(&signaling_thread_checker_) = nullptr; bool received_first_packet_ RTC_GUARDED_BY(&signaling_thread_checker_) = false; + + bool cached_track_should_receive_ RTC_GUARDED_BY(&signaling_thread_checker_); const int attachment_id_; rtc::scoped_refptr frame_decryptor_ RTC_GUARDED_BY(worker_thread_); @@ -177,6 +185,7 @@ class VideoRtpReceiver : public RtpReceiverInternal { // or switched. bool saved_generate_keyframe_ RTC_GUARDED_BY(worker_thread_) = false; bool saved_encoded_sink_enabled_ RTC_GUARDED_BY(worker_thread_) = false; + const rtc::scoped_refptr worker_thread_safety_; }; } // namespace webrtc diff --git a/pc/video_track.cc b/pc/video_track.cc index 571e49807c..d7d2871663 100644 --- a/pc/video_track.cc +++ b/pc/video_track.cc @@ -18,6 +18,7 @@ #include "rtc_base/checks.h" #include "rtc_base/location.h" #include "rtc_base/ref_counted_object.h" +#include "rtc_base/logging.h" namespace webrtc { @@ -78,6 +79,19 @@ VideoTrackSourceInterface* VideoTrack::GetSourceInternal() const { return video_source_->internal(); } +void VideoTrack::set_should_receive(bool receive) { + RTC_DCHECK_RUN_ON(&signaling_thread_); + if (should_receive_ == receive) + return; + should_receive_ = receive; + Notifier::FireOnChanged(); +} + +bool VideoTrack::should_receive() const { + RTC_DCHECK_RUN_ON(&signaling_thread_); + return should_receive_; +} + VideoTrackInterface::ContentHint VideoTrack::content_hint() const { RTC_DCHECK_RUN_ON(&signaling_thread_); return content_hint_; diff --git a/pc/video_track.h b/pc/video_track.h index 13a51c454b..b56c64ef20 100644 --- a/pc/video_track.h +++ b/pc/video_track.h @@ -48,6 +48,9 @@ class VideoTrack : public MediaStreamTrack, void RequestRefreshFrame() override; VideoTrackSourceInterface* GetSource() const override; + void set_should_receive(bool should_receive) override; + bool should_receive() const override; + ContentHint content_hint() const override; void set_content_hint(ContentHint hint) override; bool set_enabled(bool enable) override; @@ -81,6 +84,7 @@ class VideoTrack : public MediaStreamTrack, // be queried without blocking on the worker thread by callers that don't // use an api proxy to call the `enabled()` method. bool enabled_w_ RTC_GUARDED_BY(worker_thread_) = true; + bool should_receive_ RTC_GUARDED_BY(signaling_thread_) = true; }; } // namespace webrtc diff --git a/sdk/android/api/org/webrtc/VideoTrack.java b/sdk/android/api/org/webrtc/VideoTrack.java index 5593d424f3..bd5022f39a 100644 --- a/sdk/android/api/org/webrtc/VideoTrack.java +++ b/sdk/android/api/org/webrtc/VideoTrack.java @@ -54,6 +54,24 @@ public void removeSink(VideoSink sink) { } } + /** + * For a remote video track, starts/stops receiving the video stream. + * + * If this is a local video track, this is a no-op. + */ + public void setShouldReceive(boolean shouldReceive){ + nativeSetShouldReceive(getNativeMediaStreamTrack(), shouldReceive); + } + + /** + * The current receive status for a remote video track. + * + * This has no meaning for a local video track. + */ + public boolean shouldReceive(){ + return nativeGetShouldReceive(getNativeMediaStreamTrack()); + } + @Override public void dispose() { for (long nativeSink : sinks.values()) { @@ -73,4 +91,6 @@ long getNativeVideoTrack() { private static native void nativeRemoveSink(long track, long nativeSink); private static native long nativeWrapSink(VideoSink sink); private static native void nativeFreeSink(long sink); + private static native void nativeSetShouldReceive(long track, boolean shouldReceive); + private static native boolean nativeGetShouldReceive(long track); } diff --git a/sdk/android/src/jni/video_track.cc b/sdk/android/src/jni/video_track.cc index 70bedc12cf..783ede0d44 100644 --- a/sdk/android/src/jni/video_track.cc +++ b/sdk/android/src/jni/video_track.cc @@ -45,5 +45,16 @@ static void JNI_VideoTrack_FreeSink(JNIEnv* jni, delete reinterpret_cast*>(j_native_sink); } +static void JNI_VideoTrack_SetShouldReceive(JNIEnv* jni, + jlong j_native_track, + jboolean should_receive) { + reinterpret_cast(j_native_track)->set_should_receive(should_receive); +} + +static jboolean JNI_VideoTrack_GetShouldReceive(JNIEnv* jni, + jlong j_native_track) { + return reinterpret_cast(j_native_track)->should_receive(); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.h b/sdk/objc/api/peerconnection/RTCVideoTrack.h index 5382b7169f..56d25c1568 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.h +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.h @@ -25,6 +25,9 @@ RTC_OBJC_EXPORT /** The video source for this video track. */ @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCVideoSource) *source; +/** The receive state, if this is a remote video track. */ +@property(nonatomic, assign) BOOL shouldReceive; + - (instancetype)init NS_UNAVAILABLE; /** Register a renderer that will render all frames received on this track. */ diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.mm b/sdk/objc/api/peerconnection/RTCVideoTrack.mm index 8944d26c13..8764c797dc 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.mm +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.mm @@ -68,6 +68,14 @@ - (void)dealloc { return _source; } +- (BOOL)shouldReceive { + return self.nativeVideoTrack->should_receive(); +} + +- (void)setShouldReceive:(BOOL)shouldReceive { + self.nativeVideoTrack->set_should_receive(shouldReceive); +} + - (void)addRenderer:(id)renderer { // Make sure we don't have this renderer yet. for (RTCVideoRendererAdapter *adapter in _adapters) { diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index 2cf4ba5ba8..144491bf54 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -361,6 +361,7 @@ void VideoReceiveStream2::Start() { return; } + frame_buffer_->Start(); const bool protected_by_fec = config_.rtp.protected_by_flexfec || rtp_video_stream_receiver_.IsUlpfecEnabled();