diff --git a/api/media_stream_interface.cc b/api/media_stream_interface.cc index e07907917b..5362522262 100644 --- a/api/media_stream_interface.cc +++ b/api/media_stream_interface.cc @@ -18,6 +18,10 @@ const char* const MediaStreamTrackInterface::kVideoKind = const char* const MediaStreamTrackInterface::kAudioKind = cricket::kMediaTypeAudio; +bool VideoTrackInterface::should_receive() const { + return true; +} + VideoTrackInterface::ContentHint VideoTrackInterface::content_hint() const { return ContentHint::kNone; } diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h index ad497d943b..65685bfead 100644 --- a/api/media_stream_interface.h +++ b/api/media_stream_interface.h @@ -188,6 +188,8 @@ class RTC_EXPORT VideoTrackInterface virtual VideoTrackSourceInterface* GetSource() const = 0; + virtual void set_should_receive(bool should_receive) {} + virtual bool should_receive() const; virtual ContentHint content_hint() const; virtual void set_content_hint(ContentHint hint) {} diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 2607a7a4aa..95e7e961e0 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -897,6 +897,9 @@ class VideoMediaChannel : public MediaChannel, public Delayable { virtual void GenerateKeyFrame(uint32_t ssrc) = 0; virtual std::vector GetSources(uint32_t ssrc) const = 0; + + virtual void StartReceive(uint32_t ssrc) {} + virtual void StopReceive(uint32_t ssrc) {} }; // Info about data received in DataMediaChannel. For use in diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index f497d9046e..39e179ae13 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -949,6 +949,24 @@ void WebRtcVideoChannel::RequestEncoderSwitch( << format.ToString() << " not negotiated."; } +void WebRtcVideoChannel::StartReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StartStream(); +} + +void WebRtcVideoChannel::StopReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StopStream(); +} + bool WebRtcVideoChannel::ApplyChangedParams( const ChangedSendParameters& changed_params) { RTC_DCHECK_RUN_ON(&thread_checker_); @@ -3017,6 +3035,17 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters( } } +void WebRtcVideoChannel::WebRtcVideoReceiveStream::StartStream(){ + if (stream_) { + stream_->Start(); + } +} +void WebRtcVideoChannel::WebRtcVideoReceiveStream::StopStream(){ + if (stream_) { + stream_->Stop(); + } +} + void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() { absl::optional base_minimum_playout_delay_ms; absl::optional recording_state; diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index 90d824a55b..16858dac0a 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -245,7 +245,8 @@ class WebRtcVideoChannel : public VideoMediaChannel, uint32_t ssrc, rtc::scoped_refptr frame_transformer) override; - + void StartReceive(uint32_t ssrc) override; + void StopReceive(uint32_t ssrc) override; private: class WebRtcVideoReceiveStream; @@ -479,6 +480,9 @@ class WebRtcVideoChannel : public VideoMediaChannel, void SetDepacketizerToDecoderFrameTransformer( rtc::scoped_refptr frame_transformer); + + void StartStream(); + void StopStream(); private: void RecreateWebRtcVideoStream(); diff --git a/modules/video_coding/frame_buffer2.cc b/modules/video_coding/frame_buffer2.cc index b7ae0f3e94..de1631bee8 100644 --- a/modules/video_coding/frame_buffer2.cc +++ b/modules/video_coding/frame_buffer2.cc @@ -356,6 +356,13 @@ void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { protection_mode_ = mode; } +void FrameBuffer::Start() { + TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); + MutexLock lock(&mutex_); + if (!stopped_) + return; + stopped_ = false; +} void FrameBuffer::Stop() { TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); MutexLock lock(&mutex_); diff --git a/modules/video_coding/frame_buffer2.h b/modules/video_coding/frame_buffer2.h index c2a3394bad..47a080ba4a 100644 --- a/modules/video_coding/frame_buffer2.h +++ b/modules/video_coding/frame_buffer2.h @@ -78,6 +78,8 @@ class FrameBuffer { // Stop the frame buffer, causing any sleeping thread in NextFrame to // return immediately. void Stop(); + // Unstop the frame buffer, re-allowing new frames to be inserted and read. + void Start(); // Updates the RTT for jitter buffer estimation. void UpdateRtt(int64_t rtt_ms); diff --git a/pc/media_stream_track_proxy.h b/pc/media_stream_track_proxy.h index f563137c77..2f741f9ff3 100644 --- a/pc/media_stream_track_proxy.h +++ b/pc/media_stream_track_proxy.h @@ -44,16 +44,18 @@ PROXY_PRIMARY_THREAD_DESTRUCTOR() BYPASS_PROXY_CONSTMETHOD0(std::string, kind) BYPASS_PROXY_CONSTMETHOD0(std::string, id) PROXY_SECONDARY_CONSTMETHOD0(TrackState, state) -PROXY_SECONDARY_CONSTMETHOD0(bool, enabled) -PROXY_SECONDARY_METHOD1(bool, set_enabled, bool) -PROXY_SECONDARY_CONSTMETHOD0(ContentHint, content_hint) -PROXY_SECONDARY_METHOD1(void, set_content_hint, ContentHint) +PROXY_CONSTMETHOD0(bool, enabled) +PROXY_METHOD1(bool, set_enabled, bool) +PROXY_CONSTMETHOD0(ContentHint, content_hint) +PROXY_METHOD1(void, set_content_hint, ContentHint) PROXY_SECONDARY_METHOD2(void, AddOrUpdateSink, rtc::VideoSinkInterface*, const rtc::VideoSinkWants&) PROXY_SECONDARY_METHOD1(void, RemoveSink, rtc::VideoSinkInterface*) BYPASS_PROXY_CONSTMETHOD0(VideoTrackSourceInterface*, GetSource) +PROXY_CONSTMETHOD0(bool, should_receive) +PROXY_METHOD1(void, set_should_receive, bool) PROXY_METHOD1(void, RegisterObserver, ObserverInterface*) PROXY_METHOD1(void, UnregisterObserver, ObserverInterface*) diff --git a/pc/rtp_sender.cc b/pc/rtp_sender.cc index d4286371be..8f025e491a 100644 --- a/pc/rtp_sender.cc +++ b/pc/rtp_sender.cc @@ -584,10 +584,13 @@ VideoRtpSender::~VideoRtpSender() { } void VideoRtpSender::OnChanged() { + // Running on the signaling thread. TRACE_EVENT0("webrtc", "VideoRtpSender::OnChanged"); RTC_DCHECK(!stopped_); - if (cached_track_content_hint_ != video_track()->content_hint()) { - cached_track_content_hint_ = video_track()->content_hint(); + + auto content_hint = video_track()->content_hint(); + if (cached_track_content_hint_ != content_hint) { + cached_track_content_hint_ = content_hint; if (can_send_track()) { SetSend(); } diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index 8db4d9f02f..ae249d15ad 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -21,6 +21,7 @@ #include "rtc_base/checks.h" #include "rtc_base/location.h" #include "rtc_base/logging.h" +#include "rtc_base/task_utils/to_queued_task.h" namespace webrtc { @@ -46,9 +47,12 @@ VideoRtpReceiver::VideoRtpReceiver( worker_thread, source_), worker_thread))), - attachment_id_(GenerateUniqueId()) { + cached_track_should_receive_(track_->should_receive()), + attachment_id_(GenerateUniqueId()), + worker_thread_safety_(PendingTaskSafetyFlag::CreateDetachedInactive()) { RTC_DCHECK(worker_thread_); SetStreams(streams); + track_->RegisterObserver(this); RTC_DCHECK_EQ(source_->state(), MediaSourceInterface::kLive); } @@ -137,6 +141,39 @@ void VideoRtpReceiver::StopAndEndTrack() { track_->internal()->set_ended(); } +void VideoRtpReceiver::OnChanged() { + RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + if (cached_track_should_receive_ != track_->should_receive()) { + cached_track_should_receive_ = track_->should_receive(); + worker_thread_->PostTask(ToQueuedTask( + worker_thread_safety_, + [this, receive = cached_track_should_receive_]() { + RTC_DCHECK_RUN_ON(worker_thread_); + if(receive) { + StartMediaChannel(); + } else { + StopMediaChannel(); + } + })); + } +} + +void VideoRtpReceiver::StartMediaChannel() { + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StartReceive(ssrc_.value_or(0)); + OnGenerateKeyFrame(); +} +void VideoRtpReceiver::StopMediaChannel() { + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StopReceive(ssrc_.value_or(0)); +} + void VideoRtpReceiver::RestartMediaChannel(absl::optional ssrc) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); @@ -235,6 +272,7 @@ void VideoRtpReceiver::set_transport( void VideoRtpReceiver::SetStreams( const std::vector>& streams) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + // Remove remote track from any streams that are going away. for (const auto& existing_stream : streams_) { bool removed = true; @@ -301,13 +339,14 @@ void VideoRtpReceiver::SetMediaChannel(cricket::MediaChannel* media_channel) { void VideoRtpReceiver::SetMediaChannel_w(cricket::MediaChannel* media_channel) { if (media_channel == media_channel_) return; - bool encoded_sink_enabled = saved_encoded_sink_enabled_; if (encoded_sink_enabled && media_channel_) { // Turn off the old sink, if any. SetEncodedSinkEnabled(false); } + media_channel ? worker_thread_safety_->SetAlive() + : worker_thread_safety_->SetNotAlive(); media_channel_ = static_cast(media_channel); if (media_channel_) { diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index b5381860b3..67c8effef2 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -43,7 +43,8 @@ namespace webrtc { -class VideoRtpReceiver : public RtpReceiverInternal { +class VideoRtpReceiver : public RtpReceiverInternal, + public ObserverInterface { public: // An SSRC of 0 will create a receiver that will match the first SSRC it // sees. Must be called on signaling thread. @@ -61,6 +62,9 @@ class VideoRtpReceiver : public RtpReceiverInternal { rtc::scoped_refptr video_track() const { return track_; } + // ObserverInterface implementation + void OnChanged() override; + // RtpReceiverInterface implementation rtc::scoped_refptr track() const override { return track_; @@ -111,6 +115,8 @@ class VideoRtpReceiver : public RtpReceiverInternal { std::vector GetSources() const override; private: + void StartMediaChannel(); + void StopMediaChannel(); void RestartMediaChannel(absl::optional ssrc); void SetSink(rtc::VideoSinkInterface* sink) RTC_RUN_ON(worker_thread_); @@ -165,6 +171,8 @@ class VideoRtpReceiver : public RtpReceiverInternal { RTC_GUARDED_BY(&signaling_thread_checker_) = nullptr; bool received_first_packet_ RTC_GUARDED_BY(&signaling_thread_checker_) = false; + + bool cached_track_should_receive_ RTC_GUARDED_BY(&signaling_thread_checker_); const int attachment_id_; rtc::scoped_refptr frame_decryptor_ RTC_GUARDED_BY(worker_thread_); @@ -180,6 +188,7 @@ class VideoRtpReceiver : public RtpReceiverInternal { // or switched. bool saved_generate_keyframe_ RTC_GUARDED_BY(worker_thread_) = false; bool saved_encoded_sink_enabled_ RTC_GUARDED_BY(worker_thread_) = false; + const rtc::scoped_refptr worker_thread_safety_; }; } // namespace webrtc diff --git a/pc/video_track.cc b/pc/video_track.cc index d0246faa87..f6c6974a35 100644 --- a/pc/video_track.cc +++ b/pc/video_track.cc @@ -19,6 +19,7 @@ #include "rtc_base/checks.h" #include "rtc_base/location.h" #include "rtc_base/ref_counted_object.h" +#include "rtc_base/logging.h" namespace webrtc { @@ -53,7 +54,7 @@ void VideoTrack::AddOrUpdateSink(rtc::VideoSinkInterface* sink, RTC_DCHECK_RUN_ON(worker_thread_); VideoSourceBaseGuarded::AddOrUpdateSink(sink, wants); rtc::VideoSinkWants modified_wants = wants; - modified_wants.black_frames = !enabled(); + modified_wants.black_frames = !enabled_w_; video_source_->AddOrUpdateSink(sink, modified_wants); } @@ -68,13 +69,26 @@ VideoTrackSourceInterface* VideoTrack::GetSource() const { return video_source_.get(); } +void VideoTrack::set_should_receive(bool receive) { + RTC_DCHECK_RUN_ON(&signaling_thread_); + if (should_receive_ == receive) + return; + should_receive_ = receive; + Notifier::FireOnChanged(); +} + +bool VideoTrack::should_receive() const { + RTC_DCHECK_RUN_ON(&signaling_thread_); + return should_receive_; +} + VideoTrackInterface::ContentHint VideoTrack::content_hint() const { - RTC_DCHECK_RUN_ON(worker_thread_); + RTC_DCHECK_RUN_ON(&signaling_thread_); return content_hint_; } void VideoTrack::set_content_hint(ContentHint hint) { - RTC_DCHECK_RUN_ON(worker_thread_); + RTC_DCHECK_RUN_ON(&signaling_thread_); if (content_hint_ == hint) return; content_hint_ = hint; @@ -82,17 +96,29 @@ void VideoTrack::set_content_hint(ContentHint hint) { } bool VideoTrack::set_enabled(bool enable) { - RTC_DCHECK_RUN_ON(worker_thread_); - for (auto& sink_pair : sink_pairs()) { - rtc::VideoSinkWants modified_wants = sink_pair.wants; - modified_wants.black_frames = !enable; - video_source_->AddOrUpdateSink(sink_pair.sink, modified_wants); - } - return MediaStreamTrack::set_enabled(enable); + RTC_DCHECK_RUN_ON(&signaling_thread_); + + bool ret = MediaStreamTrack::set_enabled(enable); + + worker_thread_->Invoke(RTC_FROM_HERE, [&]() { + RTC_DCHECK_RUN_ON(worker_thread_); + enabled_w_ = enable; + for (auto& sink_pair : sink_pairs()) { + rtc::VideoSinkWants modified_wants = sink_pair.wants; + modified_wants.black_frames = !enable; + video_source_->AddOrUpdateSink(sink_pair.sink, modified_wants); + } + }); + + return ret; } bool VideoTrack::enabled() const { - RTC_DCHECK_RUN_ON(worker_thread_); + if (worker_thread_->IsCurrent()) { + RTC_DCHECK_RUN_ON(worker_thread_); + return enabled_w_; + } + RTC_DCHECK_RUN_ON(&signaling_thread_); return MediaStreamTrack::enabled(); } @@ -103,15 +129,9 @@ MediaStreamTrackInterface::TrackState VideoTrack::state() const { void VideoTrack::OnChanged() { RTC_DCHECK_RUN_ON(&signaling_thread_); - worker_thread_->Invoke( - RTC_FROM_HERE, [this, state = video_source_->state()]() { - // TODO(tommi): Calling set_state() this way isn't ideal since we're - // currently blocking the signaling thread and set_state() may - // internally fire notifications via `FireOnChanged()` which may further - // amplify the blocking effect on the signaling thread. - rtc::Thread::ScopedDisallowBlockingCalls no_blocking_calls; - set_state(state == MediaSourceInterface::kEnded ? kEnded : kLive); - }); + rtc::Thread::ScopedDisallowBlockingCalls no_blocking_calls; + MediaSourceInterface::SourceState state = video_source_->state(); + set_state(state == MediaSourceInterface::kEnded ? kEnded : kLive); } rtc::scoped_refptr VideoTrack::Create( diff --git a/pc/video_track.h b/pc/video_track.h index 49deaee76a..2e93fc2604 100644 --- a/pc/video_track.h +++ b/pc/video_track.h @@ -40,6 +40,9 @@ class VideoTrack : public MediaStreamTrack, void RemoveSink(rtc::VideoSinkInterface* sink) override; VideoTrackSourceInterface* GetSource() const override; + void set_should_receive(bool should_receive) override; + bool should_receive() const override; + ContentHint content_hint() const override; void set_content_hint(ContentHint hint) override; bool set_enabled(bool enable) override; @@ -60,7 +63,13 @@ class VideoTrack : public MediaStreamTrack, RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker signaling_thread_; rtc::Thread* const worker_thread_; const rtc::scoped_refptr video_source_; - ContentHint content_hint_ RTC_GUARDED_BY(worker_thread_); + ContentHint content_hint_ RTC_GUARDED_BY(signaling_thread_); + // Cached `enabled` state for the worker thread. This is kept in sync with + // the state maintained on the signaling thread via set_enabled() but can + // be queried without blocking on the worker thread by callers that don't + // use an api proxy to call the `enabled()` method. + bool enabled_w_ RTC_GUARDED_BY(worker_thread_) = true; + bool should_receive_ RTC_GUARDED_BY(signaling_thread_) = true; }; } // namespace webrtc diff --git a/sdk/android/api/org/webrtc/VideoTrack.java b/sdk/android/api/org/webrtc/VideoTrack.java index 5593d424f3..bd5022f39a 100644 --- a/sdk/android/api/org/webrtc/VideoTrack.java +++ b/sdk/android/api/org/webrtc/VideoTrack.java @@ -54,6 +54,24 @@ public void removeSink(VideoSink sink) { } } + /** + * For a remote video track, starts/stops receiving the video stream. + * + * If this is a local video track, this is a no-op. + */ + public void setShouldReceive(boolean shouldReceive){ + nativeSetShouldReceive(getNativeMediaStreamTrack(), shouldReceive); + } + + /** + * The current receive status for a remote video track. + * + * This has no meaning for a local video track. + */ + public boolean shouldReceive(){ + return nativeGetShouldReceive(getNativeMediaStreamTrack()); + } + @Override public void dispose() { for (long nativeSink : sinks.values()) { @@ -73,4 +91,6 @@ long getNativeVideoTrack() { private static native void nativeRemoveSink(long track, long nativeSink); private static native long nativeWrapSink(VideoSink sink); private static native void nativeFreeSink(long sink); + private static native void nativeSetShouldReceive(long track, boolean shouldReceive); + private static native boolean nativeGetShouldReceive(long track); } diff --git a/sdk/android/src/jni/video_track.cc b/sdk/android/src/jni/video_track.cc index 70bedc12cf..783ede0d44 100644 --- a/sdk/android/src/jni/video_track.cc +++ b/sdk/android/src/jni/video_track.cc @@ -45,5 +45,16 @@ static void JNI_VideoTrack_FreeSink(JNIEnv* jni, delete reinterpret_cast*>(j_native_sink); } +static void JNI_VideoTrack_SetShouldReceive(JNIEnv* jni, + jlong j_native_track, + jboolean should_receive) { + reinterpret_cast(j_native_track)->set_should_receive(should_receive); +} + +static jboolean JNI_VideoTrack_GetShouldReceive(JNIEnv* jni, + jlong j_native_track) { + return reinterpret_cast(j_native_track)->should_receive(); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.h b/sdk/objc/api/peerconnection/RTCVideoTrack.h index 5382b7169f..56d25c1568 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.h +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.h @@ -25,6 +25,9 @@ RTC_OBJC_EXPORT /** The video source for this video track. */ @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCVideoSource) *source; +/** The receive state, if this is a remote video track. */ +@property(nonatomic, assign) BOOL shouldReceive; + - (instancetype)init NS_UNAVAILABLE; /** Register a renderer that will render all frames received on this track. */ diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.mm b/sdk/objc/api/peerconnection/RTCVideoTrack.mm index 3f38dd51a9..a06fa59ae5 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.mm +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.mm @@ -69,6 +69,14 @@ - (void)dealloc { return _source; } +- (BOOL)shouldReceive { + return self.nativeVideoTrack->should_receive(); +} + +- (void)setShouldReceive:(BOOL)shouldReceive { + self.nativeVideoTrack->set_should_receive(shouldReceive); +} + - (void)addRenderer:(id)renderer { // Make sure we don't have this renderer yet. for (RTCVideoRendererAdapter *adapter in _adapters) { diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index 0776a40b35..120644a6fe 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -337,6 +337,7 @@ void VideoReceiveStream2::Start() { return; } + frame_buffer_->Start(); const bool protected_by_fec = config_.rtp.protected_by_flexfec || rtp_video_stream_receiver_.IsUlpfecEnabled();