From fa328bc2a37a64318f6cce82cc01bc9f9113c636 Mon Sep 17 00:00:00 2001 From: Maria Tverdostup Date: Thu, 16 Jul 2020 10:21:52 -0700 Subject: [PATCH 01/11] local commit to save state of nack test work --- package.json | 2 +- src/ShmTransport.ts | 2 +- src/Transport.ts | 7 ++++++- worker/include/RTC/ShmConsumer.hpp | 3 +-- worker/src/RTC/Producer.cpp | 7 +++++-- worker/src/RTC/RtpStreamRecv.cpp | 4 ++-- worker/src/RTC/ShmConsumer.cpp | 18 +++++++++--------- worker/src/RTC/Transport.cpp | 4 ++-- 8 files changed, 27 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index b967a9b48f..d8be3b2bc3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@livelyvideo/mediasoup", - "version": "3.5.7-shm40", + "version": "3.5.7-shm45", "description": "Cutting Edge WebRTC Video Conferencing", "contributors": [ "Iñaki Baz Castillo (https://inakibaz.me)", diff --git a/src/ShmTransport.ts b/src/ShmTransport.ts index fdb53f4b1c..868bb0cd35 100644 --- a/src/ShmTransport.ts +++ b/src/ShmTransport.ts @@ -126,7 +126,7 @@ export class ShmTransport extends Transport } /** - * Get PipeTransport stats. + * Get ShmTransport stats. * * @override */ diff --git a/src/Transport.ts b/src/Transport.ts index 5c2f2cac6c..7834b009b5 100644 --- a/src/Transport.ts +++ b/src/Transport.ts @@ -401,7 +401,7 @@ export class Transport extends EnhancedEventEmitter }: ProducerOptions ): Promise { - logger.debug('produce()'); + logger.debug('produce() rtpParameters=%o', rtpParameters); if (id && this._producers.has(id)) throw new TypeError(`a Producer with same id "${id}" already exists`); @@ -446,15 +446,20 @@ export class Transport extends EnhancedEventEmitter } const routerRtpCapabilities = this._getRouterRtpCapabilities(); + logger.debug("produce(): routerRtpCapabilities=%o", routerRtpCapabilities); // This may throw. const rtpMapping = ortc.getProducerRtpParametersMapping( rtpParameters, routerRtpCapabilities); + logger.debug("produce(): rtpMapping=%o", rtpMapping); + // This may throw. const consumableRtpParameters = ortc.getConsumableRtpParameters( kind, rtpParameters, routerRtpCapabilities, rtpMapping); + logger.debug("produce(): consumableRtpParameters=%o", consumableRtpParameters); + const internal = { ...this._internal, producerId: id || uuidv4() }; const reqData = { kind, rtpParameters, rtpMapping, keyFrameRequestDelay, paused }; diff --git a/worker/include/RTC/ShmConsumer.hpp b/worker/include/RTC/ShmConsumer.hpp index 59d0d6fdf8..6f39eac41a 100644 --- a/worker/include/RTC/ShmConsumer.hpp +++ b/worker/include/RTC/ShmConsumer.hpp @@ -54,10 +54,9 @@ namespace RTC bool WritePacketToShm(RTC::RtpPacket* packet); bool VideoOrientationChanged(RTC::RtpPacket* packet); -/* Uncomment for NACK test simulation + //Uncomment for NACK test simulation bool TestNACK(RTC::RtpPacket* packet); uint64_t lastNACKTestTs {0}; -*/ /* Pure virtual methods inherited from RtpStreamSend::Listener. */ public: diff --git a/worker/src/RTC/Producer.cpp b/worker/src/RTC/Producer.cpp index 55a9de8b29..039f7f3aaf 100644 --- a/worker/src/RTC/Producer.cpp +++ b/worker/src/RTC/Producer.cpp @@ -43,6 +43,9 @@ namespace RTC // This may throw. this->rtpParameters = RTC::RtpParameters(*jsonRtpParametersIt); + std::string s = jsonRtpParametersIt->dump(); + MS_DEBUG_TAG(rtp,"L@@K Producer ctor has RtpParameters: {%s}L@@K", s.c_str()); + // Evaluate type. this->type = RTC::RtpParameters::GetType(this->rtpParameters); @@ -613,7 +616,7 @@ namespace RTC result = ReceiveRtpPacketResult::RETRANSMISSION; isRtx = true; - MS_DEBUG_TAG(rtp, "Retransmitted packet received [ssrc:%" PRIu32 " seq:%" PRIu16 " ts:%" PRIu32 "]", + MS_DEBUG_TAG(rtp, "L@@KL@@K Retransmitted packet received [ssrc:%" PRIu32 " seq:%" PRIu16 " ts:%" PRIu32 "]", packet->GetSsrc(),packet->GetSequenceNumber(), packet->GetTimestamp()); // Process the packet. @@ -1081,7 +1084,7 @@ namespace RTC for (auto& fb : mediaCodec.rtcpFeedback) { - MS_DEBUG_2TAGS(rtp, rtcp, "mediaCodec.rtcpFeedback: type=%s parameter=%s", fb.type.c_str(), fb.parameter.c_str()); + MS_DEBUG_2TAGS(rtp, rtcp, "L@@KL@@K mediaCodec.rtcpFeedback: type=%s parameter=%s", fb.type.c_str(), fb.parameter.c_str()); if (!params.useNack && fb.type == "nack" && fb.parameter == "") { diff --git a/worker/src/RTC/RtpStreamRecv.cpp b/worker/src/RTC/RtpStreamRecv.cpp index 71adc045ba..789a0ab81d 100644 --- a/worker/src/RTC/RtpStreamRecv.cpp +++ b/worker/src/RTC/RtpStreamRecv.cpp @@ -190,7 +190,7 @@ namespace RTC this->nackGenerator.reset(new RTC::NackGenerator(this)); } else { - MS_DEBUG_TAG(rtp,"RtpStreamRecv::params.useNack is false, NACK feature disabled"); + MS_DEBUG_TAG(rtp,"L@@K RtpStreamRecv::params.useNack is false, NACK feature disabled"); } // Run the RTP inactivity periodic timer (unless DTX is enabled). if (!this->params.useDtx) @@ -828,7 +828,7 @@ namespace RTC packet.Serialize(RTC::RTCP::Buffer); - // MS_DEBUG_TAG(rtp, "Serialize NACKed RTCP packet and call OnRtpStreamSendRtcpPacket"); + MS_DEBUG_TAG(rtp, "L@@KL@@K Serialize NACKed RTCP packet and call OnRtpStreamSendRtcpPacket"); // Notify the listener. static_cast(this->listener)->OnRtpStreamSendRtcpPacket(this, &packet); } diff --git a/worker/src/RTC/ShmConsumer.cpp b/worker/src/RTC/ShmConsumer.cpp index 7e964cb2ad..48f5caadfd 100644 --- a/worker/src/RTC/ShmConsumer.cpp +++ b/worker/src/RTC/ShmConsumer.cpp @@ -36,11 +36,11 @@ namespace RTC this->shmCtx = shmCtx; -/* - Uncomment for NACK test simulation + +// Uncomment for NACK test simulation uint64_t nowTs = DepLibUV::GetTimeMs(); this->lastNACKTestTs = nowTs; -*/ + CreateRtpStream(); } @@ -262,15 +262,15 @@ namespace RTC packet->GetTimestamp(), origSeq); } -/* - Uncomment for NACK test simulation + + // Uncomment for NACK test simulation if (this->TestNACK(packet)) { MS_DEBUG_TAG(rtp, "Pretend NACK for packet ssrc:%" PRIu32 ", seq:%" PRIu16 " ts: %" PRIu32 " and wait for retransmission", packet->GetSsrc(), packet->GetSequenceNumber(), packet->GetTimestamp()); return; } -*/ + // Process the packet. if (this->WritePacketToShm(packet)) { @@ -293,8 +293,8 @@ namespace RTC packet->SetSequenceNumber(origSeq); } -/* -Uncomment for NACK test simulation + + //Uncomment for NACK test simulation bool ShmConsumer::TestNACK(RTC::RtpPacket* packet) { if (this->GetKind() != RTC::Media::Kind::VIDEO) @@ -331,7 +331,7 @@ Uncomment for NACK test simulation return true; } - */ + bool ShmConsumer::VideoOrientationChanged(RTC::RtpPacket* packet) { diff --git a/worker/src/RTC/Transport.cpp b/worker/src/RTC/Transport.cpp index 23f6764a93..2604f8f440 100644 --- a/worker/src/RTC/Transport.cpp +++ b/worker/src/RTC/Transport.cpp @@ -1421,7 +1421,7 @@ namespace RTC break; case RTC::Producer::ReceiveRtpPacketResult::RETRANSMISSION: this->recvRtxTransmission.Update(packet); - // MS_DEBUG_TAG(rtp, "recvRtxTransmission.GetPacketCount()=%d", this->recvRtxTransmission.GetPacketCount()); + MS_DEBUG_TAG(rtp, "L@@KL@@K recvRtxTransmission.GetPacketCount()=%d", this->recvRtxTransmission.GetPacketCount()); break; default:; } @@ -2387,7 +2387,7 @@ namespace RTC this->sendRtxTransmission.Update(packet); - // MS_DEBUG_TAG(rtp, "sendRtxTransmission.GetPacketCount()=%d", this->sendRtxTransmission.GetPacketCount()); + MS_DEBUG_TAG(rtp, "L@@KL@@K sendRtxTransmission.GetPacketCount()=%d", this->sendRtxTransmission.GetPacketCount()); } inline void Transport::OnConsumerKeyFrameRequested(RTC::Consumer* consumer, uint32_t mappedSsrc) From cec9f52e234d4a70e2702e3a97f51de6da87232d Mon Sep 17 00:00:00 2001 From: Maria Tverdostup Date: Wed, 29 Jul 2020 14:39:15 -0700 Subject: [PATCH 02/11] Park changes: working shm consumer video queue and NACK-related logging which has to be removed --- package.json | 2 +- worker/include/DepLibSfuShm.hpp | 281 +++++----- worker/include/RTC/RtpStreamRecv.hpp | 4 + worker/include/RTC/ShmConsumer.hpp | 2 +- worker/include/RTC/ShmWriterVideoBuffer.hpp | 40 ++ worker/src/DepLibSfuShm.cpp | 581 +++++++++++++------- worker/src/RTC/Producer.cpp | 4 +- worker/src/RTC/RtpStream.cpp | 12 + worker/src/RTC/RtpStreamRecv.cpp | 12 +- worker/src/RTC/RtxStream.cpp | 12 + worker/src/RTC/ShmConsumer.cpp | 194 +++---- worker/src/RTC/Transport.cpp | 4 +- 12 files changed, 725 insertions(+), 423 deletions(-) create mode 100644 worker/include/RTC/ShmWriterVideoBuffer.hpp diff --git a/package.json b/package.json index d8be3b2bc3..8d3312f76f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@livelyvideo/mediasoup", - "version": "3.5.7-shm45", + "version": "3.5.7-shm70", "description": "Cutting Edge WebRTC Video Conferencing", "contributors": [ "Iñaki Baz Castillo (https://inakibaz.me)", diff --git a/worker/include/DepLibSfuShm.hpp b/worker/include/DepLibSfuShm.hpp index 5ca482329f..9b5c571728 100644 --- a/worker/include/DepLibSfuShm.hpp +++ b/worker/include/DepLibSfuShm.hpp @@ -8,16 +8,16 @@ extern "C" } #include #include +#include #include #define UINT64_UNSET ((uint64_t)-1) #define MAX_SEQ_DELTA 100 #define MAX_PTS_DELTA (90000*10) +#define MTU_SIZE 1500 - -class DepLibSfuShm +namespace DepLibSfuShm { -public: enum ShmWriterStatus { SHM_WRT_UNDEFINED = 1, // default SHM_WRT_CLOSED, // writer closed. TODO: implement whatever sets this state... producer paused or closed? transport closed? @@ -34,6 +34,35 @@ class DepLibSfuShm UNDEFINED }; + // Need this to keep a buffer of roughly 2-3 video frames + enum ShmQueueStatus + { + SHM_Q_PKT_CANWRITETHRU = 99, // queue was empty, the pkt represents whole video frame or it is an aggregate, its seqId is (last_seqId+1), won't add pkt to queue + SHM_Q_PKT_TOO_OLD = 100, // timestamp too old, don't queue + SHM_Q_PKT_QUEUED_OK = 101, // added pkt data into the queue + SHM_Q_PKT_DEQUEUED_OK = 102, // wrote some data into shm on dequeueing + SHM_Q_PKT_DEQUEUED_NOTHING = 103, // nothing was written into shm on dequeueing call + SHM_Q_PKT_WAIT_FOR_NACK = 104, // stopped dequeueing because waiting for retransmission + SHM_Q_PKT_CHUNKSTART_MISSING = 105, // start of fragment chunk missing, dropped the whole chunk + SHM_Q_PKT_SHMWRITE_ERR = 106 // shm writer returned some err, check the logs for details + }; + + struct ShmQueueItem + { + // Cloned data + sfushm_av_frame_frag_t chunk; + // Memory to hold the cloned pkt data, won't be larger than MTU (should match RTC::MtuSize) + uint8_t store[MTU_SIZE]; + // Whether this is chunk's fragment + bool isChunkFragment{ false }; + // Whether this is chunk's starting fragment (or whole chunk) + bool isChunkStart{ false }; + // Whether this is chunk's ending fragment (or whole chunk) + bool isChunkEnd{ false }; + //TODO: see that it has dtor + ShmQueueItem(sfushm_av_frame_frag_t* data, bool isfragment, bool isfragmentstart, bool isfragmentend); + }; + // Contains shm configuration, writer context (if initialized), writer status class SfuShmCtx { @@ -44,27 +73,32 @@ class DepLibSfuShm void InitializeShmWriterCtx(std::string shm, std::string log, int level, int stdio); void CloseShmWriterCtx(); - ShmWriterStatus Status() const { return this->wrt_status; } // (this->wrt_ctx != nullptr ? this->wrt_status : SHM_WRT_UNDEFINED); } + ShmWriterStatus Status() const { return this->wrt_status; } uint32_t AudioSsrc() const { return (this->wrt_init.conf.channels[0].audio == 1) ? this->wrt_init.conf.channels[0].ssrc : 0; } uint32_t VideoSsrc() const { return (this->wrt_init.conf.channels[1].video == 1) ? this->wrt_init.conf.channels[1].ssrc : 0; } - ShmWriterStatus SetSsrcInShmConf(uint32_t ssrc, DepLibSfuShm::ShmChunkType kind); - - uint64_t AdjustPktTs(uint64_t ts, DepLibSfuShm::ShmChunkType kind); - uint64_t AdjustPktSeq(uint64_t seq, DepLibSfuShm::ShmChunkType kind); - void UpdatePktStat(uint64_t seq, uint64_t ts, DepLibSfuShm::ShmChunkType kind); - bool IsSeqUnset(DepLibSfuShm::ShmChunkType kind) const; - bool IsTsUnset(DepLibSfuShm::ShmChunkType kind) const; - uint64_t LastTs(DepLibSfuShm::ShmChunkType kind) const; - uint64_t LastSeq(DepLibSfuShm::ShmChunkType kind) const; - - int WriteChunk(sfushm_av_frame_frag_t* data, DepLibSfuShm::ShmChunkType kind = DepLibSfuShm::ShmChunkType::UNDEFINED, uint32_t ssrc = 0); - int WriteRtcpSenderReportTs(uint64_t lastSenderReportNtpMs, uint32_t lastSenderReporTs, DepLibSfuShm::ShmChunkType kind); + ShmWriterStatus SetSsrcInShmConf(uint32_t ssrc, ShmChunkType kind); + + uint64_t AdjustPktTs(uint64_t ts, ShmChunkType kind); + uint64_t AdjustPktSeq(uint64_t seq, ShmChunkType kind); + void UpdatePktStat(uint64_t seq, uint64_t ts, ShmChunkType kind); + bool IsSeqUnset(ShmChunkType kind) const; + bool IsTsUnset(ShmChunkType kind) const; + uint64_t LastTs(ShmChunkType kind) const; + uint64_t LastSeq(ShmChunkType kind) const; + + int WriteRtpDataToShm(ShmChunkType type, sfushm_av_frame_frag_t* data, bool isChunkFragment = false); + int WriteRtcpSenderReportTs(uint64_t lastSenderReportNtpMs, uint32_t lastSenderReporTs, ShmChunkType kind); int WriteStreamMeta(std::string metadata, std::string shm); int WriteVideoOrientation(uint16_t rotation); bool IsError(int err_code); const char* GetErrorString(int err_code); + + private: + ShmQueueStatus Enqueue( sfushm_av_frame_frag_t* data, bool isChunkFragment); + ShmQueueStatus Dequeue(); + int WriteChunk(sfushm_av_frame_frag_t* data, ShmChunkType kind = ShmChunkType::UNDEFINED); public: std::string stream_name; @@ -83,141 +117,140 @@ class DepLibSfuShm private: sfushm_av_writer_init_t wrt_init; ShmWriterStatus wrt_status; - static std::unordered_map errToString; + std::list videoPktBuffer; // Video frames queue: newest items (by seqId) added at the end of queue, oldest are read from the front }; -}; - -// Inline methods + // Inline methods -inline uint64_t DepLibSfuShm::SfuShmCtx::AdjustPktTs(uint64_t ts, DepLibSfuShm::ShmChunkType kind) -{ - uint64_t last_ts; - switch(kind) + inline uint64_t SfuShmCtx::AdjustPktTs(uint64_t ts, ShmChunkType kind) { - case DepLibSfuShm::ShmChunkType::VIDEO: - last_ts = this->last_ts_v; - break; - - case DepLibSfuShm::ShmChunkType::AUDIO: - last_ts = this->last_ts_a; - break; - - default: - return ts; // do nothing + uint64_t last_ts; + switch(kind) + { + case ShmChunkType::VIDEO: + last_ts = this->last_ts_v; + break; + + case ShmChunkType::AUDIO: + last_ts = this->last_ts_a; + break; + + default: + return ts; // do nothing + } + + if (last_ts != UINT64_UNSET) { + sfushm_av_adjust_for_overflow_32_64(last_ts, &ts, MAX_PTS_DELTA); + } + return ts; } - if (last_ts != UINT64_UNSET) { - sfushm_av_adjust_for_overflow_32_64(last_ts, &ts, MAX_PTS_DELTA); + inline uint64_t SfuShmCtx::AdjustPktSeq(uint64_t seq, ShmChunkType kind) + { + uint64_t last_seq; + switch(kind) + { + case ShmChunkType::VIDEO: + last_seq = this->last_seq_v; + break; + + case ShmChunkType::AUDIO: + last_seq = this->last_seq_a; + break; + + default: + return seq; // do nothing + } + + if (last_seq != UINT64_UNSET) { + sfushm_av_adjust_for_overflow_16_64(last_seq, &seq, MAX_SEQ_DELTA); + } + return seq; } - return ts; -} -inline uint64_t DepLibSfuShm::SfuShmCtx::AdjustPktSeq(uint64_t seq, DepLibSfuShm::ShmChunkType kind) -{ - uint64_t last_seq; - switch(kind) + inline void SfuShmCtx::UpdatePktStat(uint64_t seq, uint64_t ts, ShmChunkType kind) { - case DepLibSfuShm::ShmChunkType::VIDEO: - last_seq = this->last_seq_v; - break; - - case DepLibSfuShm::ShmChunkType::AUDIO: - last_seq = this->last_seq_a; - break; - - default: - return seq; // do nothing + //No checks, just update values since we have just written that pkt's data into shm + switch(kind) + { + case ShmChunkType::VIDEO: + this->last_seq_v = seq; + this->last_ts_v = ts; + break; + + case ShmChunkType::AUDIO: + this->last_seq_a = seq; + this->last_ts_a = ts; + break; + + default: + break; // TODO: will need stats on other pkt types + } } - if (last_seq != UINT64_UNSET) { - sfushm_av_adjust_for_overflow_16_64(last_seq, &seq, MAX_SEQ_DELTA); - } - return seq; -} -inline void DepLibSfuShm::SfuShmCtx::UpdatePktStat(uint64_t seq, uint64_t ts, DepLibSfuShm::ShmChunkType kind) -{ - //No checks, just update values since we have just written that pkt's data into shm - switch(kind) + inline bool SfuShmCtx::IsSeqUnset(ShmChunkType kind) const { - case DepLibSfuShm::ShmChunkType::VIDEO: - this->last_seq_v = seq; - this->last_ts_v = ts; - break; - - case DepLibSfuShm::ShmChunkType::AUDIO: - this->last_seq_a = seq; - this->last_ts_a = ts; - break; - - default: - break; // TODO: will need stats on other pkt types + if (kind == ShmChunkType::VIDEO) + return (this->last_seq_v == UINT64_UNSET); + else if (kind == ShmChunkType::AUDIO) + return (this->last_seq_a == UINT64_UNSET); + else + return true; // no support for other pkt types yet } -} - - -inline bool DepLibSfuShm::SfuShmCtx::IsSeqUnset(DepLibSfuShm::ShmChunkType kind) const -{ - if (kind == DepLibSfuShm::ShmChunkType::VIDEO) - return (this->last_seq_v == UINT64_UNSET); - else if (kind == DepLibSfuShm::ShmChunkType::AUDIO) - return (this->last_seq_a == UINT64_UNSET); - else - return true; // no support for other pkt types yet -} -inline bool DepLibSfuShm::SfuShmCtx::IsTsUnset(DepLibSfuShm::ShmChunkType kind) const -{ - if (kind == DepLibSfuShm::ShmChunkType::VIDEO) - return (this->last_ts_v == UINT64_UNSET); - else if (kind == DepLibSfuShm::ShmChunkType::AUDIO) - return (this->last_ts_a == UINT64_UNSET); - else - return true; // no support for other pkt types yet -} + inline bool SfuShmCtx::IsTsUnset(ShmChunkType kind) const + { + if (kind == ShmChunkType::VIDEO) + return (this->last_ts_v == UINT64_UNSET); + else if (kind == ShmChunkType::AUDIO) + return (this->last_ts_a == UINT64_UNSET); + else + return true; // no support for other pkt types yet + } -inline uint64_t DepLibSfuShm::SfuShmCtx::LastTs(DepLibSfuShm::ShmChunkType kind) const -{ - if (kind == DepLibSfuShm::ShmChunkType::VIDEO) - return this->last_ts_v; - else if (kind == DepLibSfuShm::ShmChunkType::AUDIO) - return this->last_ts_a; - else - return UINT64_UNSET; // no support for other pkt types yet -} + inline uint64_t SfuShmCtx::LastTs(ShmChunkType kind) const + { + if (kind == ShmChunkType::VIDEO) + return this->last_ts_v; + else if (kind == ShmChunkType::AUDIO) + return this->last_ts_a; + else + return UINT64_UNSET; // no support for other pkt types yet + } -inline uint64_t DepLibSfuShm::SfuShmCtx::LastSeq(DepLibSfuShm::ShmChunkType kind) const -{ - if (kind == DepLibSfuShm::ShmChunkType::VIDEO) - return this->last_seq_v; - else if (kind == DepLibSfuShm::ShmChunkType::AUDIO) - return this->last_seq_a; - else - return UINT64_UNSET; // no support for other pkt types yet -} + inline uint64_t SfuShmCtx::LastSeq(ShmChunkType kind) const + { + if (kind == ShmChunkType::VIDEO) + return this->last_seq_v; + else if (kind == ShmChunkType::AUDIO) + return this->last_seq_a; + else + return UINT64_UNSET; // no support for other pkt types yet + } -inline bool DepLibSfuShm::SfuShmCtx::IsError(int err_code) -{ - return (err_code != SFUSHM_AV_OK && err_code != SFUSHM_AV_AGAIN); -} + inline bool SfuShmCtx::IsError(int err_code) + { + return (err_code != SFUSHM_AV_OK && err_code != SFUSHM_AV_AGAIN); + } -inline const char* DepLibSfuShm::SfuShmCtx::GetErrorString(int err_code) -{ - static const char* unknownErr = "unknown SFUSHM error"; + inline const char* SfuShmCtx::GetErrorString(int err_code) + { + static const char* unknownErr = "unknown SFUSHM error"; - auto it = DepLibSfuShm::SfuShmCtx::errToString.find(err_code); + auto it = SfuShmCtx::errToString.find(err_code); - if (it == DepLibSfuShm::SfuShmCtx::errToString.end()) - return unknownErr; + if (it == SfuShmCtx::errToString.end()) + return unknownErr; - return it->second; -} + return it->second; + } +} // namespace DepLibSfuShm #endif // DEP_LIBSFUSHM_HPP diff --git a/worker/include/RTC/RtpStreamRecv.hpp b/worker/include/RTC/RtpStreamRecv.hpp index f86530f4d7..61dfcf0dc1 100644 --- a/worker/include/RTC/RtpStreamRecv.hpp +++ b/worker/include/RTC/RtpStreamRecv.hpp @@ -72,6 +72,10 @@ namespace RTC { return this->transmissionCounter.GetLayerBitrate(nowMs, spatialLayer, temporalLayer); } + uint32_t GetJitter() + { + return this->jitter; + } private: void CalculateJitter(uint32_t rtpTimestamp); diff --git a/worker/include/RTC/ShmConsumer.hpp b/worker/include/RTC/ShmConsumer.hpp index 6f39eac41a..88a4eced6d 100644 --- a/worker/include/RTC/ShmConsumer.hpp +++ b/worker/include/RTC/ShmConsumer.hpp @@ -72,7 +72,7 @@ namespace RTC bool keyFrameSupported{ false }; bool syncRequired{ false }; - bool srReceived{ false }; // do not send until SR received + bool srReceived{ false }; // do not write into shm until SR received RTC::SeqManager rtpSeqManager; diff --git a/worker/include/RTC/ShmWriterVideoBuffer.hpp b/worker/include/RTC/ShmWriterVideoBuffer.hpp new file mode 100644 index 0000000000..e7a9268db9 --- /dev/null +++ b/worker/include/RTC/ShmWriterVideoBuffer.hpp @@ -0,0 +1,40 @@ +#ifndef MS_RTC_SHM_WRITER_VIDEO_BUFFER_HPP +#define MS_RTC_SHM_WRITER_VIDEO_BUFFER_HPP + +#include "RTC/RtpPacket.hpp" +#include + +namespace RTC +{ + class ShmWriterVideoBuffer // public RTC::RtpStreamSend::Listener + { + public: + struct VideoBufferItem + { + // Cloned packet. TODO: maybe null if currently missing + RTC::RtpPacket* packet{ nullptr }; + // Memory to hold the cloned packet (with extra space for RTX encoding). + uint8_t store[RTC::MtuSize + 100]; + //if it is key frame (or part of key frame in case of fragmented pic) + bool keyFrame; + bool picBegin; + bool picEnd; + bool NaluBegin; + bool NaluEnd; + // written out to shm or discarded; either way, this item can now be removed from the buffer + bool done; + //TODO: keep timestamps and seqIds separately, or rely on packet field? + }; + //ShmWriterVideoBuffer(); + + private: + uint64_t oldestTs; + uint64_t newestTs; + uint16_t oldestSeqId; + uint16_t newestSeqId; + uint32_t ssrc; + std::vector buffer; // TODO: maybe map by seqId? + }; +} + +#endif //MS_RTC_SHM_WRITER_VIDEO_BUFFER_HPP \ No newline at end of file diff --git a/worker/src/DepLibSfuShm.cpp b/worker/src/DepLibSfuShm.cpp index 554a6ecb52..d8ed260790 100644 --- a/worker/src/DepLibSfuShm.cpp +++ b/worker/src/DepLibSfuShm.cpp @@ -9,244 +9,447 @@ #include "DepLibUV.hpp" #include -std::unordered_map DepLibSfuShm::SfuShmCtx::errToString = - { - { 0, "success (SFUSHM_AV_OK)" }, - { -1, "error (SFUSHM_AV_ERR)" }, - { -2, "again (SFUSHM_AV_AGAIN)" } - }; - - -DepLibSfuShm::SfuShmCtx::~SfuShmCtx() -{ - // Call if writer is not closed - if (SHM_WRT_CLOSED != wrt_status && SHM_WRT_UNDEFINED != wrt_status) - { - sfushm_av_close_writer(wrt_ctx, 0); //TODO: smth else at the last param - } -} +namespace DepLibSfuShm { + + std::unordered_map DepLibSfuShm::SfuShmCtx::errToString = + { + { 0, "success (SFUSHM_AV_OK)" }, + { -1, "error (SFUSHM_AV_ERR)" }, + { -2, "again (SFUSHM_AV_AGAIN)" }, + { 100, "too old pkt ts (SHM_Q_PKT_TOO_OLD)"} + }; -void DepLibSfuShm::SfuShmCtx::CloseShmWriterCtx() -{ - // Call if writer is not closed - if (SHM_WRT_CLOSED != wrt_status) + static constexpr uint64_t MaxVideoPktDelay{ 9000 }; // 100ms in samples (90000 sample rate) TODO: any way to better estimate this one? + + + SfuShmCtx::~SfuShmCtx() { - sfushm_av_close_writer(wrt_ctx, 0); //TODO: smth else at the last param + // Call if writer is not closed + if (SHM_WRT_CLOSED != wrt_status && SHM_WRT_UNDEFINED != wrt_status) + { + sfushm_av_close_writer(wrt_ctx, 0); + } } - wrt_status = SHM_WRT_CLOSED; - memset( &wrt_init, 0, sizeof(sfushm_av_writer_init_t)); -} + void SfuShmCtx::CloseShmWriterCtx() + { + // Call if writer is not closed + if (SHM_WRT_CLOSED != wrt_status) + { + sfushm_av_close_writer(wrt_ctx, 0); + } -DepLibSfuShm::ShmWriterStatus DepLibSfuShm::SfuShmCtx::SetSsrcInShmConf(uint32_t ssrc, DepLibSfuShm::ShmChunkType kind) -{ - //Assuming that ssrc does not change, shm writer is initialized, nothing else to do - if (SHM_WRT_READY == this->Status()) - return SHM_WRT_READY; - - switch(kind) { - case DepLibSfuShm::ShmChunkType::AUDIO: - ssrc_a = ssrc; - this->wrt_init.conf.channels[0].audio = 1; - this->wrt_init.conf.channels[0].ssrc = ssrc; - this->wrt_status = (this->wrt_status == SHM_WRT_AUDIO_CHNL_CONF_MISSING) ? SHM_WRT_READY : SHM_WRT_VIDEO_CHNL_CONF_MISSING; - break; + wrt_status = SHM_WRT_CLOSED; + memset( &wrt_init, 0, sizeof(sfushm_av_writer_init_t)); + } - case DepLibSfuShm::ShmChunkType::VIDEO: - ssrc_v = ssrc; - this->wrt_init.conf.channels[1].video = 1; - this->wrt_init.conf.channels[1].ssrc = ssrc; - this->wrt_status = (this->wrt_status == SHM_WRT_VIDEO_CHNL_CONF_MISSING) ? SHM_WRT_READY : SHM_WRT_AUDIO_CHNL_CONF_MISSING; - break; - default: + ShmWriterStatus SfuShmCtx::SetSsrcInShmConf(uint32_t ssrc, DepLibSfuShm::ShmChunkType kind) + { + //Assuming that ssrc does not change, shm writer is initialized, nothing else to do + if (SHM_WRT_READY == this->Status()) + return SHM_WRT_READY; + + switch(kind) { + case DepLibSfuShm::ShmChunkType::AUDIO: + ssrc_a = ssrc; + this->wrt_init.conf.channels[0].audio = 1; + this->wrt_init.conf.channels[0].ssrc = ssrc; + this->wrt_status = (this->wrt_status == SHM_WRT_AUDIO_CHNL_CONF_MISSING) ? SHM_WRT_READY : SHM_WRT_VIDEO_CHNL_CONF_MISSING; + break; + + case DepLibSfuShm::ShmChunkType::VIDEO: + ssrc_v = ssrc; + this->wrt_init.conf.channels[1].video = 1; + this->wrt_init.conf.channels[1].ssrc = ssrc; + this->wrt_status = (this->wrt_status == SHM_WRT_VIDEO_CHNL_CONF_MISSING) ? SHM_WRT_READY : SHM_WRT_AUDIO_CHNL_CONF_MISSING; + break; + + default: + return this->Status(); + } + + if (this->wrt_status == SHM_WRT_READY) { + int err = SFUSHM_AV_OK; + if ((err = sfushm_av_open_writer( &wrt_init, &wrt_ctx)) != SFUSHM_AV_OK) { + MS_DEBUG_TAG(rtp, "FAILED in sfushm_av_open_writer() to initialize sfu shm %s with error %s", this->wrt_init.stream_name, GetErrorString(err)); + wrt_status = SHM_WRT_UNDEFINED; + } return this->Status(); + } + + return this->Status(); // if shm was not initialized as a result, return false } - if (this->wrt_status == SHM_WRT_READY) { - int err = SFUSHM_AV_OK; - if ((err = sfushm_av_open_writer( &wrt_init, &wrt_ctx)) != SFUSHM_AV_OK) { - MS_DEBUG_TAG(rtp, "FAILED in sfushm_av_open_writer() to initialize sfu shm %s with error %s", this->wrt_init.stream_name, GetErrorString(err)); - wrt_status = SHM_WRT_UNDEFINED; - } - return this->Status(); + + void SfuShmCtx::InitializeShmWriterCtx(std::string shm, std::string log, int level, int stdio) + { + MS_TRACE(); + + memset( &wrt_init, 0, sizeof(sfushm_av_writer_init_t)); + + stream_name.assign(shm); + log_name.assign(log); + + wrt_init.stream_name = const_cast(stream_name.c_str()); + wrt_init.stats_win_size = 300; + wrt_init.conf.log_file_name = log_name.c_str(); + wrt_init.conf.log_level = level; + wrt_init.conf.redirect_stdio = stdio; + + wrt_init.conf.channels[0].target_buf_ms = 20000; + wrt_init.conf.channels[0].target_kbps = 128; + wrt_init.conf.channels[0].ssrc = 0; + wrt_init.conf.channels[0].sample_rate = 48000; + wrt_init.conf.channels[0].num_chn = 2; + wrt_init.conf.channels[0].codec_id = SFUSHM_AV_AUDIO_CODEC_OPUS; + wrt_init.conf.channels[0].video = 0; + wrt_init.conf.channels[0].audio = 1; + + wrt_init.conf.channels[1].target_buf_ms = 20000; + wrt_init.conf.channels[1].target_kbps = 2500; + wrt_init.conf.channels[1].ssrc = 0; + wrt_init.conf.channels[1].sample_rate = 90000; + wrt_init.conf.channels[1].codec_id = SFUSHM_AV_VIDEO_CODEC_H264; + wrt_init.conf.channels[1].video = 1; + wrt_init.conf.channels[1].audio = 0; } - return this->Status(); // if shm was not initialized as a result, return false -} + ShmQueueItem::ShmQueueItem(sfushm_av_frame_frag_t* chunk, bool isfragment, bool isfragmentstart, bool isfragmentend) + : isChunkFragment(isfragment), isChunkStart(isfragmentstart), isChunkEnd(isfragmentend) + { + std::memcpy(&(this->chunk), chunk, sizeof(sfushm_av_frame_frag_t)); + this->chunk.data = (uint8_t*)std::addressof(this->store); + std::memcpy(this->chunk.data, chunk->data, chunk->len); + } -void DepLibSfuShm::SfuShmCtx::InitializeShmWriterCtx(std::string shm, std::string log, int level, int stdio) -{ - MS_TRACE(); - memset( &wrt_init, 0, sizeof(sfushm_av_writer_init_t)); + ShmQueueStatus SfuShmCtx::Enqueue(sfushm_av_frame_frag_t* data, bool isChunkFragment) + { + uint64_t ts = data->rtp_time; + uint64_t seq = data->first_rtp_seq; + bool isChunkStart = data->begin; + bool isChunkEnd = data->end; + + // If queue is empty, seqid is in order, and pkt is not a fragment, there is no need to enqueue + if (this->videoPktBuffer.empty() + && !isChunkFragment + && (LastSeq(ShmChunkType::VIDEO) == UINT64_UNSET + || data->first_rtp_seq - 1 == LastSeq(ShmChunkType::VIDEO))) + { + MS_DEBUG_TAG(rtp, "SHM_Q_PKT_CANWRITETHRU seqid=%" PRIu64 "(%" PRIu64 ") ts=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", + data->first_rtp_seq, LastSeq(ShmChunkType::VIDEO), data->rtp_time, LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); + return ShmQueueStatus::SHM_Q_PKT_CANWRITETHRU; + } + + // Reject old pkt + if (LastSeq(ShmChunkType::VIDEO) != UINT64_UNSET + && LastTs(ShmChunkType::VIDEO) > ts + && LastTs(ShmChunkType::VIDEO) - ts > MaxVideoPktDelay) + { + MS_DEBUG_TAG(rtp, "SHM_Q_PKT_TOO_OLD seqid=%" PRIu64 " ts=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", + data->first_rtp_seq, data->rtp_time, LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); + return ShmQueueStatus::SHM_Q_PKT_TOO_OLD; + } + + // Enqueue pkt, newest at the end + auto it = this->videoPktBuffer.begin(); + while (it != this->videoPktBuffer.end()) + { + if (seq >= it->chunk.first_rtp_seq) // incoming seqId is newer, move further to the end + ++it; + } + it = videoPktBuffer.emplace(it, data, isChunkFragment, isChunkStart, isChunkEnd); + + MS_DEBUG_TAG(rtp, "Successully enqueued data seqId=%" PRIu64 "%s%s%s len=%zu ts=%" PRIu64 " lastSeq=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", + seq, isChunkFragment ? " isFragment" : "", isChunkStart ? " chunkStart" : "", isChunkEnd ? " chunkEnd" : "", it->chunk.len, + ts, LastSeq(ShmChunkType::VIDEO), LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); - stream_name.assign(shm); - log_name.assign(log); + return SHM_Q_PKT_QUEUED_OK; + } - wrt_init.stream_name = const_cast(stream_name.c_str()); - wrt_init.stats_win_size = 300; - wrt_init.conf.log_file_name = log_name.c_str(); - wrt_init.conf.log_level = level; - wrt_init.conf.redirect_stdio = stdio; - // TODO: initialize with some different values? or keep these? - // At this points ssrc values are still not known - wrt_init.conf.channels[0].target_buf_ms = 20000; - wrt_init.conf.channels[0].target_kbps = 128; - wrt_init.conf.channels[0].ssrc = 0; - wrt_init.conf.channels[0].sample_rate = 48000; - wrt_init.conf.channels[0].num_chn = 2; - wrt_init.conf.channels[0].codec_id = SFUSHM_AV_AUDIO_CODEC_OPUS; - wrt_init.conf.channels[0].video = 0; - wrt_init.conf.channels[0].audio = 1; - - wrt_init.conf.channels[1].target_buf_ms = 20000; - wrt_init.conf.channels[1].target_kbps = 2500; - wrt_init.conf.channels[1].ssrc = 0; - wrt_init.conf.channels[1].sample_rate = 90000; - wrt_init.conf.channels[1].codec_id = SFUSHM_AV_VIDEO_CODEC_H264; - wrt_init.conf.channels[1].video = 1; - wrt_init.conf.channels[1].audio = 0; -} - - -int DepLibSfuShm::SfuShmCtx::WriteChunk(sfushm_av_frame_frag_t* data, DepLibSfuShm::ShmChunkType kind, uint32_t ssrc) -{ - int err; - - if (Status() != SHM_WRT_READY) + ShmQueueStatus SfuShmCtx::Dequeue() { - return 0; + ShmQueueStatus ret = SHM_Q_PKT_DEQUEUED_NOTHING; + int err; + + if (this->videoPktBuffer.empty()) + return SHM_Q_PKT_DEQUEUED_NOTHING; + + // TODO: this algorithm tracks previous seq id in a queue, confirm that it is always monotonous + std::_List_iterator chunkStartIt; + + auto it = this->videoPktBuffer.begin(); + uint64_t prev_seq = it->chunk.first_rtp_seq - 1; + bool chunkStartFound = false; + + while (it != this->videoPktBuffer.end()) + { + // First, drop chunks with expired timestamps: last timestamp of a written or skipped packet + if (LastSeq(ShmChunkType::VIDEO) != UINT64_UNSET + && LastTs(ShmChunkType::VIDEO) > it->chunk.rtp_time + && LastTs(ShmChunkType::VIDEO) - it->chunk.rtp_time > MaxVideoPktDelay) // TODO: review ts check and verify that LastTs() is updated properly in each scenario + { + MS_DEBUG_TAG(rtp, "dropping pkt seqId=%" PRIu64 " ts=%" PRIu64 " LastTs=%" PRIu64 " qsize=%zu", + it->chunk.first_rtp_seq, it->chunk.rtp_time, LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); + prev_seq = it->chunk.first_rtp_seq; + it = this->videoPktBuffer.erase(it); + continue; + } + + // Hole in seqIds, exit and wait for missing pkt to be retransmitted + if (it->chunk.first_rtp_seq > prev_seq + 1) + { + MS_DEBUG_TAG(rtp, "HOLE in seqId: cur=%" PRIu64 " prev=%" PRIu64, it->chunk.first_rtp_seq, prev_seq); + return SHM_Q_PKT_WAIT_FOR_NACK; + } + + // Fragment of a chunk: start, end or middle + if (it->isChunkFragment) + { + if (it->isChunkStart) // start: remember where it is and keep iterating + { + chunkStartFound = true; + chunkStartIt = it; + prev_seq = it->chunk.first_rtp_seq; + ++it; + } + else // mid or end + { + if (!chunkStartFound) + { + // chunk incomplete, wait for retransmission + MS_DEBUG_TAG(rtp, "found fragmented chunk but no start yet: [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", + it->chunk.first_rtp_seq, it->chunk.rtp_time, videoPktBuffer.size()); + return SHM_Q_PKT_CHUNKSTART_MISSING; + } + + // write the whole chunk if we have it + if (it->isChunkEnd) + { + auto nextit = it; + uint64_t startSeqId = chunkStartIt->chunk.first_rtp_seq; + uint64_t startTs = chunkStartIt->chunk.rtp_time; + uint64_t endSeqId = it->chunk.first_rtp_seq; + uint64_t endTs = it->chunk.rtp_time; + prev_seq = it->chunk.first_rtp_seq; + + nextit++; // position past last fragment in a chunk + while (chunkStartIt != nextit) + { + MS_DEBUG_TAG(rtp, "writing fragment seq=%" PRIu64 " [%X%X%X%X]", + chunkStartIt->chunk.first_rtp_seq, + chunkStartIt->chunk.data[0],chunkStartIt->chunk.data[1],chunkStartIt->chunk.data[2],chunkStartIt->chunk.data[3]); + err = this->WriteChunk(&chunkStartIt->chunk, ShmChunkType::VIDEO); + ret = IsError(err) ? SHM_Q_PKT_SHMWRITE_ERR : SHM_Q_PKT_DEQUEUED_OK; + chunkStartIt = this->videoPktBuffer.erase(chunkStartIt); + } + MS_DEBUG_TAG(rtp, "wrote fragmented chunk! start: [seq=%" PRIu64 " ts=%" PRIu64 "] end: [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", + startSeqId, startTs, endSeqId, endTs, videoPktBuffer.size()); + chunkStartFound = false; + it = nextit; // restore iterator + } + else + { + MS_DEBUG_TAG(rtp, "skip over mid fragment [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", + it->chunk.first_rtp_seq, it->chunk.rtp_time, videoPktBuffer.size()); + prev_seq = it->chunk.first_rtp_seq; + ++it; + } + } + } + else // non-fragmented chunk + { + err = this->WriteChunk(&it->chunk, ShmChunkType::VIDEO); + MS_DEBUG_TAG(rtp, "wrote non-fragmented chunk! seqId=%" PRIu64 " ts=%" PRIu64 " qsize=%zu [%X%X%X%X]", + it->chunk.first_rtp_seq, it->chunk.rtp_time, videoPktBuffer.size(), + it->chunk.data[0],it->chunk.data[1],it->chunk.data[2],it->chunk.data[3]); + ret = IsError(err) ? SHM_Q_PKT_SHMWRITE_ERR : SHM_Q_PKT_DEQUEUED_OK; + prev_seq = it->chunk.first_rtp_seq; + it = this->videoPktBuffer.erase(it); + continue; + } + } + return ret; } - switch (kind) - { - case DepLibSfuShm::ShmChunkType::VIDEO: - err = sfushm_av_write_video(wrt_ctx, data); - break; - case DepLibSfuShm::ShmChunkType::AUDIO: - err = sfushm_av_write_audio(wrt_ctx, data); - break; + int SfuShmCtx::WriteRtpDataToShm(ShmChunkType type, sfushm_av_frame_frag_t* frag, bool isChunkFragment) + { + int err; + ShmQueueStatus st; + + if (type == ShmChunkType::VIDEO) + { + st = this->Enqueue(frag, isChunkFragment); + if (SHM_Q_PKT_CANWRITETHRU == st) // Write it into shm, no need to queue anything + { + MS_DEBUG_TAG(rtp, "wrote whole chunk seqid=%" PRIu64 " ts=%" PRIu64 " [%X%X%X%X]", + frag->first_rtp_seq, frag->rtp_time, frag->data[0],frag->data[1],frag->data[2],frag->data[3]); + + return this->WriteChunk(frag, ShmChunkType::VIDEO); + } + + if (SHM_Q_PKT_TOO_OLD == st) // Packet is too old, cannot write it + { + return SHM_Q_PKT_TOO_OLD; + } + + st = this->Dequeue(); + if (SHM_Q_PKT_DEQUEUED_OK != st) + { + MS_DEBUG_TAG(rtp,"Dequeue() returned state %d", st); + return (SHM_Q_PKT_SHMWRITE_ERR == st) ? -1 : 0; + } + } + else + { + err = this->WriteChunk(frag, ShmChunkType::AUDIO); // do not queue audio + } - //case DepLibSfuShm::ShmChunkType::RTCP: - default: - return -1; + return err; } - if (IsError(err)) + + int SfuShmCtx::WriteChunk(sfushm_av_frame_frag_t* data, ShmChunkType kind) { - MS_WARN_TAG(rtp, "ERROR writing chunk to shm: %d - %s", err, GetErrorString(err)); - return -1; - } - return 0; -} + int err; + + if (Status() != SHM_WRT_READY) + { + return 0; + } + switch (kind) + { + case ShmChunkType::VIDEO: + err = sfushm_av_write_video(wrt_ctx, data); + break; -int DepLibSfuShm::SfuShmCtx::WriteRtcpSenderReportTs(uint64_t lastSenderReportNtpMs, uint32_t lastSenderReporTs, DepLibSfuShm::ShmChunkType kind) -{ - if (Status() != SHM_WRT_READY) - { + case ShmChunkType::AUDIO: + err = sfushm_av_write_audio(wrt_ctx, data); + break; + + default: + return -1; + } + + if (IsError(err)) + { + // TODO: also log first_rtp_seq, ssrc and timestamp + MS_WARN_TAG(rtp, "ERROR writing chunk to shm: %d - %s", err, GetErrorString(err)); + return -1; + } return 0; } - uint32_t ssrc; - switch(kind) + + int SfuShmCtx::WriteRtcpSenderReportTs(uint64_t lastSenderReportNtpMs, uint32_t lastSenderReporTs, DepLibSfuShm::ShmChunkType kind) { - case DepLibSfuShm::ShmChunkType::AUDIO: - ssrc = ssrc_a; - break; + if (Status() != SHM_WRT_READY) + { + return 0; + } - case DepLibSfuShm::ShmChunkType::VIDEO: - ssrc = ssrc_v; - break; + uint32_t ssrc; + switch(kind) + { + case DepLibSfuShm::ShmChunkType::AUDIO: + ssrc = ssrc_a; + break; - default: - return 0; - } + case DepLibSfuShm::ShmChunkType::VIDEO: + ssrc = ssrc_v; + break; + + default: + return 0; + } + /* + .. c:function:: uint64_t uv_hrtime(void) + + Returns the current high-resolution real time. This is expressed in + nanoseconds. It is relative to an arbitrary time in the past. It is not + related to the time of day and therefore not subject to clock drift. The + primary use is for measuring performance between intervals. */ + auto ntp = Utils::Time::TimeMs2Ntp(lastSenderReportNtpMs); + auto ntp_sec = ntp.seconds; + auto ntp_frac = ntp.fractions; /* -.. c:function:: uint64_t uv_hrtime(void) - - Returns the current high-resolution real time. This is expressed in - nanoseconds. It is relative to an arbitrary time in the past. It is not - related to the time of day and therefore not subject to clock drift. The - primary use is for measuring performance between intervals. */ - auto ntp = Utils::Time::TimeMs2Ntp(lastSenderReportNtpMs); - auto ntp_sec = ntp.seconds; - auto ntp_frac = ntp.fractions; -/* - Uncomment for debugging - uint64_t walltime = DepLibUV::GetTimeMs(); - struct timespec clockTime; - time_t ct; - if (clock_gettime( CLOCK_REALTIME, &clockTime) == -1) { - ct = 0; + Uncomment for debugging + uint64_t walltime = DepLibUV::GetTimeMs(); + struct timespec clockTime; + time_t ct; + if (clock_gettime( CLOCK_REALTIME, &clockTime) == -1) { + ct = 0; + } + ct = clockTime.tv_sec; + uint64_t clockTimeMs = (clockTime.tv_sec * (uint64_t) 1e9 + clockTime.tv_nsec) / 1000000.0; + + MS_DEBUG_TAG(rtp, "RTCP SR: SSRC=%d ReportNTP(ms)=%" PRIu64 " RtpTs=%" PRIu32 " uv_hrtime(ms)=%" PRIu64 " clock_gettime(s)=%" PRIu64 " clock_gettime(ms)=%" PRIu64, + ssrc, + lastSenderReportNtpMs, + lastSenderReporTs, + walltime, + ct, + clockTimeMs); + */ + int err = sfushm_av_write_rtcp_sr_ts(wrt_ctx, ntp_sec, ntp_frac, lastSenderReporTs, ssrc); + + if (IsError(err)) + { + MS_WARN_TAG(rtp, "ERROR writing RTCP SR: %d - %s", err, GetErrorString(err)); + return -1; + } + + return 0; } - ct = clockTime.tv_sec; - uint64_t clockTimeMs = (clockTime.tv_sec * (uint64_t) 1e9 + clockTime.tv_nsec) / 1000000.0; - - MS_DEBUG_TAG(rtp, "RTCP SR: SSRC=%d ReportNTP(ms)=%" PRIu64 " RtpTs=%" PRIu32 " uv_hrtime(ms)=%" PRIu64 " clock_gettime(s)=%" PRIu64 " clock_gettime(ms)=%" PRIu64, - ssrc, - lastSenderReportNtpMs, - lastSenderReporTs, - walltime, - ct, - clockTimeMs); -*/ - int err = sfushm_av_write_rtcp_sr_ts(wrt_ctx, ntp_sec, ntp_frac, lastSenderReporTs, ssrc); - - if (IsError(err)) + + + int SfuShmCtx::WriteStreamMeta(std::string metadata, std::string shm) { - MS_WARN_TAG(rtp, "ERROR writing RTCP SR: %d - %s", err, GetErrorString(err)); - return -1; - } + int err; + uint8_t data[256]; // Currently this is overkill because just 1 byte will be written successfully into shm - return 0; -} + if (0 != this->stream_name.compare(shm)) + { + MS_WARN_TAG(rtp, "input metadata shm name '%s' does not match '%s'", shm.c_str(), this->stream_name.c_str()); + return -1; + } + if (metadata.length() > 255) + { + MS_WARN_TAG(rtp, "input metadata is too long: %s", metadata.c_str()); + } + std::copy(metadata.begin(), metadata.end(), data); -int DepLibSfuShm::SfuShmCtx::WriteStreamMeta(std::string metadata, std::string shm) -{ - int err; - uint8_t data[256]; // Currently this is overkill because just 1 byte will be written successfully into shm + err = sfushm_av_write_stream_metadata(wrt_ctx, data, metadata.length()); - if (0 != this->stream_name.compare(shm)) - { - MS_WARN_TAG(rtp, "input metadata shm name '%s' does not match '%s'", shm.c_str(), this->stream_name.c_str()); - return -1; - } + if (IsError(err)) + { + MS_WARN_TAG(rtp, "ERROR writing stream metadata: %d - %s", err, GetErrorString(err)); + return -1; + } - if (metadata.length() > 255) - { - MS_WARN_TAG(rtp, "input metadata is too long: %s", metadata.c_str()); + return 0; } - std::copy(metadata.begin(), metadata.end(), data); - err = sfushm_av_write_stream_metadata(wrt_ctx, data, metadata.length()); - if (IsError(err)) + int SfuShmCtx::WriteVideoOrientation(uint16_t rotation) { - MS_WARN_TAG(rtp, "ERROR writing stream metadata: %d - %s", err, GetErrorString(err)); - return -1; - } - - return 0; -} + int err = sfushm_av_write_video_rotation(wrt_ctx, rotation); -int DepLibSfuShm::SfuShmCtx::WriteVideoOrientation(uint16_t rotation) -{ - int err = sfushm_av_write_video_rotation(wrt_ctx, rotation); + if (IsError(err)) + { + MS_WARN_TAG(rtp, "ERROR writing video rotation: %d - %s", err, GetErrorString(err)); + return -1; + } - if (IsError(err)) - { - MS_WARN_TAG(rtp, "ERROR writing video rotation: %d - %s", err, GetErrorString(err)); - return -1; + return 0; } - - return 0; -} +} // namespace DepLibSfuShm \ No newline at end of file diff --git a/worker/src/RTC/Producer.cpp b/worker/src/RTC/Producer.cpp index 039f7f3aaf..cd1bed59b4 100644 --- a/worker/src/RTC/Producer.cpp +++ b/worker/src/RTC/Producer.cpp @@ -616,7 +616,7 @@ namespace RTC result = ReceiveRtpPacketResult::RETRANSMISSION; isRtx = true; - MS_DEBUG_TAG(rtp, "L@@KL@@K Retransmitted packet received [ssrc:%" PRIu32 " seq:%" PRIu16 " ts:%" PRIu32 "]", + MS_DEBUG_TAG(rtp, "L@@K Retransmitted packet received [ssrc:%" PRIu32 " seq:%" PRIu16 " ts:%" PRIu32 "]", packet->GetSsrc(),packet->GetSequenceNumber(), packet->GetTimestamp()); // Process the packet. @@ -1084,7 +1084,7 @@ namespace RTC for (auto& fb : mediaCodec.rtcpFeedback) { - MS_DEBUG_2TAGS(rtp, rtcp, "L@@KL@@K mediaCodec.rtcpFeedback: type=%s parameter=%s", fb.type.c_str(), fb.parameter.c_str()); + MS_DEBUG_2TAGS(rtp, rtcp, "L@@K mediaCodec.rtcpFeedback: type=%s parameter=%s", fb.type.c_str(), fb.parameter.c_str()); if (!params.useNack && fb.type == "nack" && fb.parameter == "") { diff --git a/worker/src/RTC/RtpStream.cpp b/worker/src/RTC/RtpStream.cpp index ac208457fb..e1b04b8ffc 100644 --- a/worker/src/RTC/RtpStream.cpp +++ b/worker/src/RTC/RtpStream.cpp @@ -205,6 +205,12 @@ namespace RTC packet->GetSsrc(), packet->GetSequenceNumber()); + MS_DEBUG_TAG( + rtp, + "L@@K too bad sequence number, re-syncing RTP [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", + packet->GetSsrc(), + packet->GetSequenceNumber()); + InitSeq(seq); this->maxPacketTs = packet->GetTimestamp(); @@ -217,6 +223,12 @@ namespace RTC "bad sequence number, ignoring packet [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", packet->GetSsrc(), packet->GetSequenceNumber()); + + MS_DEBUG_TAG( + rtp, + "L@@K bad sequence number, ignoring packet [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", + packet->GetSsrc(), + packet->GetSequenceNumber()); this->badSeq = (seq + 1) & (RtpSeqMod - 1); diff --git a/worker/src/RTC/RtpStreamRecv.cpp b/worker/src/RTC/RtpStreamRecv.cpp index 789a0ab81d..5006567891 100644 --- a/worker/src/RTC/RtpStreamRecv.cpp +++ b/worker/src/RTC/RtpStreamRecv.cpp @@ -190,8 +190,11 @@ namespace RTC this->nackGenerator.reset(new RTC::NackGenerator(this)); } else { - MS_DEBUG_TAG(rtp,"L@@K RtpStreamRecv::params.useNack is false, NACK feature disabled"); + MS_DEBUG_TAG(rtp,"L@@KL@@KL@@K RtpStreamRecv::params.useNack is false, NACK feature disabled"); } + + MS_DEBUG_TAG(rtp, "L@@KL@@KL@@K RtpStreamRecv::ctor() HasRtx is %s", HasRtx() ? "TRUE" : "FALSE"); + // Run the RTP inactivity periodic timer (unless DTX is enabled). if (!this->params.useDtx) { @@ -324,6 +327,13 @@ namespace RTC return false; } + MS_DEBUG_TAG( + rtx, + "L@@KL@@KL@@K Received RTX packet [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", + packet->GetSsrc(), + packet->GetSequenceNumber() + ); + if (HasRtx()) { if (!this->rtxStream->ReceivePacket(packet)) diff --git a/worker/src/RTC/RtxStream.cpp b/worker/src/RTC/RtxStream.cpp index fe6b0b2efb..2e1ea20c82 100644 --- a/worker/src/RTC/RtxStream.cpp +++ b/worker/src/RTC/RtxStream.cpp @@ -191,6 +191,12 @@ namespace RTC packet->GetSsrc(), packet->GetSequenceNumber()); + MS_DEBUG_TAG( + rtx, + "L@@K RTX too bad sequence number, re-syncing RTP [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", + packet->GetSsrc(), + packet->GetSequenceNumber()); + InitSeq(seq); this->maxPacketTs = packet->GetTimestamp(); @@ -204,6 +210,12 @@ namespace RTC packet->GetSsrc(), packet->GetSequenceNumber()); + MS_DEBUG_TAG( + rtx, + "L@@K RTX bad sequence number, ignoring packet [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", + packet->GetSsrc(), + packet->GetSequenceNumber()); + this->badSeq = (seq + 1) & (RtpSeqMod - 1); // Packet discarded due to late or early arriving. diff --git a/worker/src/RTC/ShmConsumer.cpp b/worker/src/RTC/ShmConsumer.cpp index 48f5caadfd..931ee8f218 100644 --- a/worker/src/RTC/ShmConsumer.cpp +++ b/worker/src/RTC/ShmConsumer.cpp @@ -244,9 +244,6 @@ namespace RTC { // Send the packet. this->listener->OnConsumerSendRtpPacket(this, packet); - - // Send the packet. - this->listener->OnConsumerSendRtpPacket(this, packet); // May emit 'trace' event. EmitTraceEventRtpAndKeyFrameTypes(packet); @@ -263,15 +260,49 @@ namespace RTC origSeq); } - // Uncomment for NACK test simulation - if (this->TestNACK(packet)) + // Check for video orientation changes + if (VideoOrientationChanged(packet)) + { + MS_DEBUG_TAG(rtp, "Video orientation changed to %d in packet[ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + this->rotation, + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); + + shmCtx->WriteVideoOrientation(this->rotation); + } + + // If we have not written any packets yet, need to configure shm writer + if (shmWriterCounter.GetPacketCount() == 0) + { + auto kind = (this->GetKind() == RTC::Media::Kind::AUDIO) ? DepLibSfuShm::ShmChunkType::AUDIO : DepLibSfuShm::ShmChunkType::VIDEO; + if (DepLibSfuShm::ShmWriterStatus::SHM_WRT_READY != shmCtx->SetSsrcInShmConf(packet->GetSsrc(), kind)) + { + return; + } + } + + // Cannot write into shm until the first RTCP Sender Report received + if (!this->srReceived) { +/* MS_DEBUG_TAG(rtp, "RTCP SR not received yet, ignoring packet[ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); */ + return; + } + +// Uncomment for NACK test simulation +/* if (this->TestNACK(packet)) { MS_DEBUG_TAG(rtp, "Pretend NACK for packet ssrc:%" PRIu32 ", seq:%" PRIu16 " ts: %" PRIu32 " and wait for retransmission", packet->GetSsrc(), packet->GetSequenceNumber(), packet->GetTimestamp()); return; } +*/ +// End of NACK test simulation - // Process the packet. + // TODO: WritePacketToShm may actually discard it instead of writing into shm. + // Is it still okay to trust its ret value to have similar semantics? if (this->WritePacketToShm(packet)) { // Increase transmission counter. @@ -294,14 +325,14 @@ namespace RTC } - //Uncomment for NACK test simulation +//Uncomment for NACK test simulation bool ShmConsumer::TestNACK(RTC::RtpPacket* packet) { if (this->GetKind() != RTC::Media::Kind::VIDEO) return false; // not video uint64_t nowTs = DepLibUV::GetTimeMs(); - if (nowTs - this->lastNACKTestTs < 300) + if (nowTs - this->lastNACKTestTs < 3000) return false; // too soon this->lastNACKTestTs = nowTs; @@ -331,6 +362,7 @@ namespace RTC return true; } +// End of NACK test simulation bool ShmConsumer::VideoOrientationChanged(RTC::RtpPacket* packet) @@ -358,43 +390,14 @@ namespace RTC return true; } } - return false; } bool ShmConsumer::WritePacketToShm(RTC::RtpPacket* packet) { MS_TRACE(); - // If we have not written any packets yet, need to configure shm writer - if (shmWriterCounter.GetPacketCount() == 0) - { - auto kind = (this->GetKind() == RTC::Media::Kind::AUDIO) ? DepLibSfuShm::ShmChunkType::AUDIO : DepLibSfuShm::ShmChunkType::VIDEO; - if (DepLibSfuShm::ShmWriterStatus::SHM_WRT_READY != shmCtx->SetSsrcInShmConf(packet->GetSsrc(), kind)) - { - return false; - } - } - - // Cannot write into shm until the first RTCP Sender Report received - if (!this->srReceived) { - MS_DEBUG_TAG(rtp, "RTCP SR not received yet, ignoring packet[ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp()); - return false; - } - - if (VideoOrientationChanged(packet)) - { - MS_DEBUG_TAG(rtp, "Video orientation changed to %d in packet[ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", - this->rotation, - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp()); - - shmCtx->WriteVideoOrientation(this->rotation); - } + bool ret = true; uint8_t const* pktdata = packet->GetData(); uint8_t const* cdata = packet->GetPayload(); uint8_t* data = const_cast(cdata); @@ -402,38 +405,34 @@ namespace RTC uint64_t ts = static_cast(packet->GetTimestamp()); uint64_t seq = static_cast(packet->GetSequenceNumber()); uint32_t ssrc = packet->GetSsrc(); - // bool isKeyFrame = packet->IsKeyFrame(); uncomment for debugging output + bool isKeyFrame = packet->IsKeyFrame(); //uncomment for debugging output std::memset(&chunk, 0, sizeof(chunk)); + ts = shmCtx->AdjustPktTs(ts, this->GetKind() == Media::Kind::VIDEO ? DepLibSfuShm::ShmChunkType::VIDEO : DepLibSfuShm::ShmChunkType::AUDIO); + seq = shmCtx->AdjustPktSeq(seq, this->GetKind() == Media::Kind::VIDEO ? DepLibSfuShm::ShmChunkType::VIDEO : DepLibSfuShm::ShmChunkType::AUDIO); + switch (this->GetKind()) { - /* Just write out the payload */ + /* Audio NALUs are always simple */ case RTC::Media::Kind::AUDIO: { - //ts = shmCtx->AdjustPktTs(ts, DepLibSfuShm::ShmChunkType::AUDIO); - seq = shmCtx->AdjustPktSeq(seq, DepLibSfuShm::ShmChunkType::AUDIO); this->chunk.data = data; this->chunk.len = len; this->chunk.rtp_time = ts; this->chunk.first_rtp_seq = this->chunk.last_rtp_seq = seq; this->chunk.ssrc = packet->GetSsrc(); this->chunk.begin = this->chunk.end = 1; - if(0 != shmCtx->WriteChunk(&chunk, DepLibSfuShm::ShmChunkType::AUDIO, packet->GetSsrc())) + if(0 != shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::AUDIO, &chunk)) { MS_WARN_TAG(rtp, "FAIL writing audio ts %" PRIu64 " seq %" PRIu64, this->chunk.rtp_time, this->chunk.first_rtp_seq); - return false; + ret = false; } - - shmCtx->UpdatePktStat(seq, ts, DepLibSfuShm::ShmChunkType::AUDIO); break; } // audio case RTC::Media::Kind::VIDEO: { - ts = shmCtx->AdjustPktTs(ts, DepLibSfuShm::ShmChunkType::VIDEO); - seq = shmCtx->AdjustPktSeq(seq, DepLibSfuShm::ShmChunkType::VIDEO); - uint8_t nal = *(data) & 0x1F; uint8_t marker = *(pktdata + 1) & 0x80; // Marker bit indicates the last or the only NALU in this packet is the end of the picture data @@ -448,7 +447,7 @@ namespace RTC break; } - //Add Annex B 0x00000001 to the begininng of this packet: overwrite 3 or 4 bytes from the PTP pkt header + //Add Annex B 0x00000001 to the beginning of this packet: overwrite 3 or 4 bytes from the PTP pkt header uint16_t chunksize = len; if (begin_picture) { @@ -478,22 +477,15 @@ namespace RTC if (nal != 1) { MS_DEBUG_TAG(rtp, "video single NALU=%d LEN=%zu ts %" PRIu64 " seq %" PRIu64 " isKeyFrame=%d begin_picture(chunk.begin)=%d marker(chunk.end)=%d lastTs=%" PRIu64, - nal, - this->chunk.len, - this->chunk.rtp_time, - this->chunk.first_rtp_seq, - isKeyFrame, - begin_picture, - marker, - shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO)); - } - */ - if (0 != shmCtx->WriteChunk(&chunk, DepLibSfuShm::ShmChunkType::VIDEO, packet->GetSsrc())) + nal, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, isKeyFrame, begin_picture, marker, shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO)); + } */ + if (0 != shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &chunk)) { MS_WARN_TAG(rtp, "FAIL writing video NALU=%d: ts %" PRIu64 " seq %" PRIu64, nal, this->chunk.rtp_time, this->chunk.first_rtp_seq); - return false; + ret=false; + break; } - } + } // Single NALU pkt else { switch (nal) { @@ -562,26 +554,17 @@ namespace RTC this->chunk.rtp_time = ts; this->chunk.first_rtp_seq = this->chunk.last_rtp_seq = seq; // TODO: is it okay to send several NALUs with same seq? this->chunk.ssrc = ssrc; - /* - if (subnal != 1) + /* if (subnal != 1) { MS_DEBUG_TAG(rtp, "video STAP-A: NAL=%d payloadlen=%" PRIu32 " nalulen=%" PRIu16 " chunklen=%" PRIu32 " ts=%" PRIu64 " seq=%" PRIu64 " lastTs=%" PRIu64 " isKeyFrame=%d chunk.begin=%d chunk.end=%d", - subnal, - len, - naluSize, - this->chunk.len, - this->chunk.rtp_time, - this->chunk.first_rtp_seq, - shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO), - isKeyFrame, - this->chunk.begin, - this->chunk.end); - } - */ - if (0 != shmCtx->WriteChunk(&chunk, DepLibSfuShm::ShmChunkType::VIDEO, ssrc)) + subnal, len, naluSize, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, + shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO), isKeyFrame, this->chunk.begin, this->chunk.end); + } */ + if (0 != shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &chunk)) { MS_WARN_TAG(rtp, "FAIL writing STAP-A pkt to shm: len %zu ts %" PRIu64 " seq %" PRIu64, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq); - return false; + ret=false; + break; } offset += chunksize; @@ -608,7 +591,8 @@ namespace RTC if (len < 3) { MS_DEBUG_TAG(rtp, "FU-A payload too short"); - return false; + ret=false; + break; } // Parse FU header octet uint8_t startBit = *(data + 1) & 0x80; // 1st bit indicates the starting fragment @@ -642,7 +626,8 @@ namespace RTC this->chunk.begin = 0; } } - else { // if not the beginning fragment, discard FU indicator and FU header + else { + // if not the beginning fragment, discard FU indicator and FU header chunksize -= 1; data += 2; this->chunk.begin = 0; @@ -654,26 +639,16 @@ namespace RTC this->chunk.first_rtp_seq = this->chunk.last_rtp_seq = seq; this->chunk.ssrc = ssrc; this->chunk.end = (endBit && marker) ? 1 : 0; - /* - if (subnal != 1) { + /* if (subnal != 1) { MS_DEBUG_TAG(rtp, "video FU-A NAL=%" PRIu8 " len=%" PRIu32 " ts=%" PRIu64 " prev_ts=%" PRIu64 " seq=%" PRIu64 " isKeyFrame=%d startBit=%" PRIu8 " endBit=%" PRIu8 " marker=%" PRIu8 " chunk.begin=%d chunk.end=%d", - subnal, - this->chunk.len, - this->chunk.rtp_time, - shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO), - this->chunk.first_rtp_seq, - isKeyFrame, - startBit, - endBit, - marker, - this->chunk.begin, - this->chunk.end); - } - */ - if (0 != shmCtx->WriteChunk(&chunk, DepLibSfuShm::ShmChunkType::VIDEO, ssrc)) + subnal, this->chunk.len, this->chunk.rtp_time, shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO), this->chunk.first_rtp_seq, + isKeyFrame, startBit, endBit, marker, this->chunk.begin, this->chunk.end); + } */ + if (0 != shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &(this->chunk), true)) { MS_WARN_TAG(rtp, "FAIL writing FU-A pkt to shm: len=%zu ts=%" PRIu64 " seq=%" PRIu64 "%s%s", this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, this->chunk.begin ? " begin" : "", this->chunk.end ? " end": ""); - return false; + ret=false; + break; } break; } @@ -683,16 +658,17 @@ namespace RTC case 29: // FU-B { MS_WARN_TAG(rtp, "Unsupported NAL unit type %u in video packet", nal); - return false; + ret=false; + break; } default: // ignore the rest { MS_DEBUG_TAG(rtp, "Unknown NAL unit type %u in video packet", nal); - return false; + ret=false; + break; } } - } - shmCtx->UpdatePktStat(seq, ts, DepLibSfuShm::ShmChunkType::VIDEO); + } // Aggregate or fragmented NALUs break; } // case video @@ -700,7 +676,19 @@ namespace RTC break; } // kind - return true; + // Update last timestamp and seqId, even if we failed to write pkt + switch (this->GetKind()) + { + case RTC::Media::Kind::AUDIO: + shmCtx->UpdatePktStat(seq, ts, DepLibSfuShm::ShmChunkType::AUDIO); + break; + case RTC::Media::Kind::VIDEO: + shmCtx->UpdatePktStat(seq, ts, DepLibSfuShm::ShmChunkType::VIDEO); + break; + default: + break; + } + return ret; } diff --git a/worker/src/RTC/Transport.cpp b/worker/src/RTC/Transport.cpp index 2604f8f440..bef8378561 100644 --- a/worker/src/RTC/Transport.cpp +++ b/worker/src/RTC/Transport.cpp @@ -1421,7 +1421,7 @@ namespace RTC break; case RTC::Producer::ReceiveRtpPacketResult::RETRANSMISSION: this->recvRtxTransmission.Update(packet); - MS_DEBUG_TAG(rtp, "L@@KL@@K recvRtxTransmission.GetPacketCount()=%d", this->recvRtxTransmission.GetPacketCount()); + MS_DEBUG_TAG(rtp, "L@@K recvRtxTransmission.GetPacketCount()=%d", this->recvRtxTransmission.GetPacketCount()); break; default:; } @@ -2387,7 +2387,7 @@ namespace RTC this->sendRtxTransmission.Update(packet); - MS_DEBUG_TAG(rtp, "L@@KL@@K sendRtxTransmission.GetPacketCount()=%d", this->sendRtxTransmission.GetPacketCount()); + MS_DEBUG_TAG(rtp, "L@@K sendRtxTransmission.GetPacketCount()=%d", this->sendRtxTransmission.GetPacketCount()); } inline void Transport::OnConsumerKeyFrameRequested(RTC::Consumer* consumer, uint32_t mappedSsrc) From a756c389657029811d9e43fe31c167d37262f8c4 Mon Sep 17 00:00:00 2001 From: Maria Tverdostup Date: Tue, 4 Aug 2020 12:31:08 -0700 Subject: [PATCH 03/11] park changes, need to switch branches --- package.json | 2 +- worker/src/RTC/ShmConsumer.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 92c622bb4c..6a368565c7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@livelyvideo/mediasoup", - "version": "3.5.7-shm71", + "version": "3.5.7-shm72", "description": "Cutting Edge WebRTC Video Conferencing", "contributors": [ "Iñaki Baz Castillo (https://inakibaz.me)", diff --git a/worker/src/RTC/ShmConsumer.cpp b/worker/src/RTC/ShmConsumer.cpp index 2506868f16..d55c61744f 100644 --- a/worker/src/RTC/ShmConsumer.cpp +++ b/worker/src/RTC/ShmConsumer.cpp @@ -335,13 +335,13 @@ namespace RTC } // Uncomment for NACK test simulation -/* if (this->TestNACK(packet)) + if (this->TestNACK(packet)) { MS_DEBUG_TAG(rtp, "Pretend NACK for packet ssrc:%" PRIu32 ", seq:%" PRIu16 " ts: %" PRIu32 " and wait for retransmission", packet->GetSsrc(), packet->GetSequenceNumber(), packet->GetTimestamp()); return; } -*/ + // End of NACK test simulation // TODO: WritePacketToShm may actually discard it instead of writing into shm. From ecead45468d081df560c49fd2ed3d1a3bab21342 Mon Sep 17 00:00:00 2001 From: Maria Tverdostup Date: Tue, 11 Aug 2020 09:51:23 -0700 Subject: [PATCH 04/11] Parking changes: queue works, NACKs are triggered but no retransmitted packets are received --- lib/Router.js | 2 +- lib/Transport.d.ts.map | 2 +- lib/Transport.js | 5 +- lib/Worker.js | 2 +- lib/index.js | 2 +- package.json | 2 +- src/Router.ts | 2 +- worker/include/DepLibSfuShm.hpp | 30 ++++----- worker/src/DepLibSfuShm.cpp | 62 +++++++----------- worker/src/RTC/PipeConsumer.cpp | 7 ++ worker/src/RTC/RtpStreamRecv.cpp | 5 +- worker/src/RTC/ShmConsumer.cpp | 107 ++++++++++++++++++++----------- worker/src/RTC/ShmTransport.cpp | 2 +- 13 files changed, 127 insertions(+), 103 deletions(-) diff --git a/lib/Router.js b/lib/Router.js index bfa4e7b6db..9ce487f06f 100644 --- a/lib/Router.js +++ b/lib/Router.js @@ -340,7 +340,7 @@ class Router extends EnhancedEventEmitter_1.EnhancedEventEmitter { * @returns {ShmTransport} */ async createShmTransport({ listenIp, shm, log, appData = {} }) { - logger.debug('createShmTransport()'); + logger.debug('createShmTransport() [shm:%o]', shm); if (!listenIp) throw new TypeError('missing listenIp'); else if (appData && typeof appData !== 'object') diff --git a/lib/Transport.d.ts.map b/lib/Transport.d.ts.map index 192ac220b9..2296f041a5 100644 --- a/lib/Transport.d.ts.map +++ b/lib/Transport.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"Transport.d.ts","sourceRoot":"","sources":["../src/Transport.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAG9D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAwB,MAAM,kBAAkB,CAAC;AAExE,MAAM,WAAW,iBAAiB;IAEjC;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,oBAAY,iBAAiB,GAAG,KAAK,GAAG,KAAK,CAAC;AAE9C,MAAM,WAAW,cAAc;IAE9B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,iBAAiB,CAAC;CAC5B;AAED;;GAEG;AACH,oBAAY,uBAAuB,GAAG,WAAW,GAAG,KAAK,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,uBAAuB;IAEvC;;OAEG;IACH,IAAI,EAAE,uBAAuB,CAAC;IAE9B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,SAAS,EAAE,IAAI,GAAG,KAAK,CAAC;IAExB;;OAEG;IACH,IAAI,EAAE,GAAG,CAAC;CACV;AAED,oBAAY,SAAS,GAAG,KAAK,GAAG,YAAY,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAIjF,qBAAa,SAAU,SAAQ,oBAAoB;IAGlD,SAAS,CAAC,QAAQ,CAAC,SAAS,EAC5B;QACC,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;KACpB,CAAC;IAGF,SAAS,CAAC,QAAQ,CAAC,KAAK,EACxB;QACC,cAAc,CAAC,EAAE,cAAc,CAAC;QAChC,SAAS,CAAC,EAAE,SAAS,CAAC;KACtB,CAAC;IAGF,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAGrC,SAAS,CAAC,OAAO,UAAS;IAG1B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAM;IAGhC,SAAS,CAAC,QAAQ,CAAC,yBAAyB,EAAE,MAAM,eAAe,CAAC;IAGpE,SAAS,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,QAAQ,CAAC;IAGtE,SAAS,CAAC,QAAQ,CAAC,oBAAoB,EAAE,CAAC,cAAc,EAAE,MAAM,KAAK,YAAY,CAAC;IAGlF,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAa;IAGjE,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAa;IAGjE,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAa;IAGzE,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAa;IAGzE,OAAO,CAAC,kBAAkB,CAAC,CAAS;IAGpC,OAAO,CAAC,oBAAoB,CAAK;IAGjC,OAAO,CAAC,cAAc,CAAC,CAAS;IAGhC,OAAO,CAAC,iBAAiB,CAAK;IAG9B,SAAS,CAAC,QAAQ,CAAC,SAAS,uBAA8B;IAE1D;;;;;;;;;OASG;gBAEF,EACC,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,OAAO,EACP,wBAAwB,EACxB,eAAe,EACf,mBAAmB,EACnB,EACD;QACC,QAAQ,EAAE,GAAG,CAAC;QACd,IAAI,EAAE,GAAG,CAAC;QACV,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,GAAG,CAAC;QACb,wBAAwB,EAAE,MAAM,eAAe,CAAC;QAChD,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,QAAQ,CAAC;QAClD,mBAAmB,EAAE,CAAC,cAAc,EAAE,MAAM,KAAK,YAAY,CAAC;KAC9D;IAgBF;;OAEG;IACH,IAAI,EAAE,IAAI,MAAM,CAGf;IAED;;OAEG;IACH,IAAI,MAAM,IAAI,OAAO,CAGpB;IAED;;OAEG;IACH,IAAI,OAAO,IAAI,GAAG,CAGjB;IAED;;OAEG;IACH,IAAI,OAAO,CAAC,OAAO,EAAE,GAAG,EAGvB;IAED;;;;;;;;OAQG;IACH,IAAI,QAAQ,IAAI,oBAAoB,CAGnC;IAED;;OAEG;IACH,KAAK,IAAI,IAAI;IAuDb;;;;;OAKG;IACH,YAAY,IAAI,IAAI;IAoDpB;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC;IAO1B;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAMhC;;;;OAIG;IAEG,OAAO,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAMzC;;OAEG;IACG,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAU3D;;OAEG;IACG,OAAO,CACZ,EACC,EAAc,EACd,IAAI,EACJ,aAAa,EACb,MAAc,EACd,oBAAoB,EACpB,OAAY,EACZ,EAAE,eAAe,GAChB,OAAO,CAAC,QAAQ,CAAC;IA8FpB;;;;OAIG;IACG,OAAO,CACZ,EACC,UAAU,EACV,eAAe,EACf,MAAc,EACd,eAAe,EACf,OAAY,EACZ,EAAE,eAAe,GAChB,OAAO,CAAC,QAAQ,CAAC;IA+EpB;;OAEG;IACG,WAAW,CAChB,EACC,EAAc,EACd,oBAAoB,EACpB,KAAU,EACV,QAAa,EACb,OAAY,EACZ,EAAE,mBAAmB,GACpB,OAAO,CAAC,YAAY,CAAC;IAyCxB;;OAEG;IACG,WAAW,CAChB,EACC,cAAc,EACd,OAAY,EACZ,EAAE,mBAAmB,GACpB,OAAO,CAAC,YAAY,CAAC;IA2DxB;;OAEG;IACG,gBAAgB,CAAC,KAAK,GAAE,uBAAuB,EAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAU5E,OAAO,CAAC,oBAAoB;CA+B5B"} \ No newline at end of file +{"version":3,"file":"Transport.d.ts","sourceRoot":"","sources":["../src/Transport.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAG9D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAwB,MAAM,kBAAkB,CAAC;AAExE,MAAM,WAAW,iBAAiB;IAEjC;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,oBAAY,iBAAiB,GAAG,KAAK,GAAG,KAAK,CAAC;AAE9C,MAAM,WAAW,cAAc;IAE9B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,iBAAiB,CAAC;CAC5B;AAED;;GAEG;AACH,oBAAY,uBAAuB,GAAG,WAAW,GAAG,KAAK,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,uBAAuB;IAEvC;;OAEG;IACH,IAAI,EAAE,uBAAuB,CAAC;IAE9B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,SAAS,EAAE,IAAI,GAAG,KAAK,CAAC;IAExB;;OAEG;IACH,IAAI,EAAE,GAAG,CAAC;CACV;AAED,oBAAY,SAAS,GAAG,KAAK,GAAG,YAAY,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAIjF,qBAAa,SAAU,SAAQ,oBAAoB;IAGlD,SAAS,CAAC,QAAQ,CAAC,SAAS,EAC5B;QACC,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;KACpB,CAAC;IAGF,SAAS,CAAC,QAAQ,CAAC,KAAK,EACxB;QACC,cAAc,CAAC,EAAE,cAAc,CAAC;QAChC,SAAS,CAAC,EAAE,SAAS,CAAC;KACtB,CAAC;IAGF,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAGrC,SAAS,CAAC,OAAO,UAAS;IAG1B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAM;IAGhC,SAAS,CAAC,QAAQ,CAAC,yBAAyB,EAAE,MAAM,eAAe,CAAC;IAGpE,SAAS,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,QAAQ,CAAC;IAGtE,SAAS,CAAC,QAAQ,CAAC,oBAAoB,EAAE,CAAC,cAAc,EAAE,MAAM,KAAK,YAAY,CAAC;IAGlF,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAa;IAGjE,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAa;IAGjE,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAa;IAGzE,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAa;IAGzE,OAAO,CAAC,kBAAkB,CAAC,CAAS;IAGpC,OAAO,CAAC,oBAAoB,CAAK;IAGjC,OAAO,CAAC,cAAc,CAAC,CAAS;IAGhC,OAAO,CAAC,iBAAiB,CAAK;IAG9B,SAAS,CAAC,QAAQ,CAAC,SAAS,uBAA8B;IAE1D;;;;;;;;;OASG;gBAEF,EACC,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,OAAO,EACP,wBAAwB,EACxB,eAAe,EACf,mBAAmB,EACnB,EACD;QACC,QAAQ,EAAE,GAAG,CAAC;QACd,IAAI,EAAE,GAAG,CAAC;QACV,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,GAAG,CAAC;QACb,wBAAwB,EAAE,MAAM,eAAe,CAAC;QAChD,eAAe,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,QAAQ,CAAC;QAClD,mBAAmB,EAAE,CAAC,cAAc,EAAE,MAAM,KAAK,YAAY,CAAC;KAC9D;IAgBF;;OAEG;IACH,IAAI,EAAE,IAAI,MAAM,CAGf;IAED;;OAEG;IACH,IAAI,MAAM,IAAI,OAAO,CAGpB;IAED;;OAEG;IACH,IAAI,OAAO,IAAI,GAAG,CAGjB;IAED;;OAEG;IACH,IAAI,OAAO,CAAC,OAAO,EAAE,GAAG,EAGvB;IAED;;;;;;;;OAQG;IACH,IAAI,QAAQ,IAAI,oBAAoB,CAGnC;IAED;;OAEG;IACH,KAAK,IAAI,IAAI;IAuDb;;;;;OAKG;IACH,YAAY,IAAI,IAAI;IAoDpB;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC;IAO1B;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAMhC;;;;OAIG;IAEG,OAAO,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAMzC;;OAEG;IACG,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAU3D;;OAEG;IACG,OAAO,CACZ,EACC,EAAc,EACd,IAAI,EACJ,aAAa,EACb,MAAc,EACd,oBAAoB,EACpB,OAAY,EACZ,EAAE,eAAe,GAChB,OAAO,CAAC,QAAQ,CAAC;IAmGpB;;;;OAIG;IACG,OAAO,CACZ,EACC,UAAU,EACV,eAAe,EACf,MAAc,EACd,eAAe,EACf,OAAY,EACZ,EAAE,eAAe,GAChB,OAAO,CAAC,QAAQ,CAAC;IA+EpB;;OAEG;IACG,WAAW,CAChB,EACC,EAAc,EACd,oBAAoB,EACpB,KAAU,EACV,QAAa,EACb,OAAY,EACZ,EAAE,mBAAmB,GACpB,OAAO,CAAC,YAAY,CAAC;IAyCxB;;OAEG;IACG,WAAW,CAChB,EACC,cAAc,EACd,OAAY,EACZ,EAAE,mBAAmB,GACpB,OAAO,CAAC,YAAY,CAAC;IA2DxB;;OAEG;IACG,gBAAgB,CAAC,KAAK,GAAE,uBAAuB,EAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAU5E,OAAO,CAAC,oBAAoB;CA+B5B"} \ No newline at end of file diff --git a/lib/Transport.js b/lib/Transport.js index 60486c03a2..e403a7dbc8 100644 --- a/lib/Transport.js +++ b/lib/Transport.js @@ -210,7 +210,7 @@ class Transport extends EnhancedEventEmitter_1.EnhancedEventEmitter { * Create a Producer. */ async produce({ id = undefined, kind, rtpParameters, paused = false, keyFrameRequestDelay, appData = {} }) { - logger.debug('produce()'); + logger.debug('produce() rtpParameters=%o', rtpParameters); if (id && this._producers.has(id)) throw new TypeError(`a Producer with same id "${id}" already exists`); else if (!['audio', 'video'].includes(kind)) @@ -243,10 +243,13 @@ class Transport extends EnhancedEventEmitter_1.EnhancedEventEmitter { rtpParameters.rtcp.cname = this._cnameForProducers; } const routerRtpCapabilities = this._getRouterRtpCapabilities(); + logger.debug("produce(): routerRtpCapabilities=%o", routerRtpCapabilities); // This may throw. const rtpMapping = ortc.getProducerRtpParametersMapping(rtpParameters, routerRtpCapabilities); + logger.debug("produce(): rtpMapping=%o", rtpMapping); // This may throw. const consumableRtpParameters = ortc.getConsumableRtpParameters(kind, rtpParameters, routerRtpCapabilities, rtpMapping); + logger.debug("produce(): consumableRtpParameters=%o", consumableRtpParameters); const internal = { ...this._internal, producerId: id || uuid_1.v4() }; const reqData = { kind, rtpParameters, rtpMapping, keyFrameRequestDelay, paused }; const status = await this._channel.request('transport.produce', internal, reqData); diff --git a/lib/Worker.js b/lib/Worker.js index 1579b46bcc..7b0ba86ff7 100644 --- a/lib/Worker.js +++ b/lib/Worker.js @@ -73,7 +73,7 @@ class Worker extends EnhancedEventEmitter_1.EnhancedEventEmitter { // options { env: { - MEDIASOUP_VERSION: '3.5.7-shm47' + MEDIASOUP_VERSION: '3.5.7-shm73' }, detached: false, // fd 0 (stdin) : Just ignore it. diff --git a/lib/index.js b/lib/index.js index e64bdd231f..41f132a308 100644 --- a/lib/index.js +++ b/lib/index.js @@ -17,7 +17,7 @@ exports.types = types; /** * Expose mediasoup version. */ -exports.version = '3.5.7-shm47'; +exports.version = '3.5.7-shm73'; /** * Expose parseScalabilityMode() function. */ diff --git a/package.json b/package.json index 6a368565c7..8fb2db11a3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@livelyvideo/mediasoup", - "version": "3.5.7-shm72", + "version": "3.5.7-shm73-rtx16", "description": "Cutting Edge WebRTC Video Conferencing", "contributors": [ "Iñaki Baz Castillo (https://inakibaz.me)", diff --git a/src/Router.ts b/src/Router.ts index 2d4fd63b35..57b326c198 100644 --- a/src/Router.ts +++ b/src/Router.ts @@ -629,7 +629,7 @@ export class Router extends EnhancedEventEmitter } : ShmTransportOptions ): Promise { - logger.debug('createShmTransport()'); + logger.debug('createShmTransport() [shm:%o]', shm); if (!listenIp) throw new TypeError('missing listenIp'); diff --git a/worker/include/DepLibSfuShm.hpp b/worker/include/DepLibSfuShm.hpp index 9b5c571728..241be459df 100644 --- a/worker/include/DepLibSfuShm.hpp +++ b/worker/include/DepLibSfuShm.hpp @@ -47,22 +47,19 @@ namespace DepLibSfuShm SHM_Q_PKT_SHMWRITE_ERR = 106 // shm writer returned some err, check the logs for details }; + // Video NALUs queue item struct ShmQueueItem { - // Cloned data - sfushm_av_frame_frag_t chunk; - // Memory to hold the cloned pkt data, won't be larger than MTU (should match RTC::MtuSize) - uint8_t store[MTU_SIZE]; - // Whether this is chunk's fragment - bool isChunkFragment{ false }; - // Whether this is chunk's starting fragment (or whole chunk) - bool isChunkStart{ false }; - // Whether this is chunk's ending fragment (or whole chunk) - bool isChunkEnd{ false }; - //TODO: see that it has dtor + sfushm_av_frame_frag_t chunk; // Cloned data representing either a whole NALU or a fragment + uint8_t store[MTU_SIZE]; // Memory to hold the cloned pkt data, won't be larger than MTU (should match RTC::MtuSize) + bool isChunkFragment{ false }; // If this is a picture's fragment (can be whole NALU or NALU's fragment) + bool isChunkStart{ false }; // The first (or only) picture's fragment + bool isChunkEnd{ false }; // Picture's last (or only) fragment + ShmQueueItem(sfushm_av_frame_frag_t* data, bool isfragment, bool isfragmentstart, bool isfragmentend); }; + // Contains shm configuration, writer context (if initialized), writer status class SfuShmCtx { @@ -105,20 +102,19 @@ namespace DepLibSfuShm std::string log_name; sfushm_av_wr_ctx_t *wrt_ctx; - uint64_t last_seq_a; // last RTP sequence processed by this input - uint64_t last_ts_a; // the timestamp of the last processed RTP message - uint64_t last_seq_v; // last RTP sequence processed by this input - uint64_t last_ts_v; // the timestamp of the last processed RTP message + uint64_t last_seq_a; // last RTP pkt sequence processed by this input + uint64_t last_ts_a; // the timestamp of the last processed RTP pkt + uint64_t last_seq_v; // last RTP pkt sequence processed by this input + uint64_t last_ts_v; // the timestamp of the last processed RTP messpktage uint32_t ssrc_v; // ssrc of audio chn uint32_t ssrc_a; // ssrc of video chn - int last_err {0}; // - private: sfushm_av_writer_init_t wrt_init; ShmWriterStatus wrt_status; static std::unordered_map errToString; std::list videoPktBuffer; // Video frames queue: newest items (by seqId) added at the end of queue, oldest are read from the front + // TODO: variable age of queued items... say, keep 3 pictures max before dropping and moving along }; // Inline methods diff --git a/worker/src/DepLibSfuShm.cpp b/worker/src/DepLibSfuShm.cpp index d8ed260790..9cc2b47807 100644 --- a/worker/src/DepLibSfuShm.cpp +++ b/worker/src/DepLibSfuShm.cpp @@ -15,8 +15,7 @@ namespace DepLibSfuShm { { { 0, "success (SFUSHM_AV_OK)" }, { -1, "error (SFUSHM_AV_ERR)" }, - { -2, "again (SFUSHM_AV_AGAIN)" }, - { 100, "too old pkt ts (SHM_Q_PKT_TOO_OLD)"} + { -2, "again (SFUSHM_AV_AGAIN)" } }; @@ -77,10 +76,9 @@ namespace DepLibSfuShm { MS_DEBUG_TAG(rtp, "FAILED in sfushm_av_open_writer() to initialize sfu shm %s with error %s", this->wrt_init.stream_name, GetErrorString(err)); wrt_status = SHM_WRT_UNDEFINED; } - return this->Status(); } - return this->Status(); // if shm was not initialized as a result, return false + return this->Status(); } @@ -127,6 +125,9 @@ namespace DepLibSfuShm { } + // seqId: strictly +1 unless NALUs came from STAP-A packet, then they are the same + // timestamps: should increment for each new picture (shm chunk), otherwise they match + // Can see two fragments coming in with the same seqId and timestamps: SPS and PPS from the same STAP-A packet are typical ShmQueueStatus SfuShmCtx::Enqueue(sfushm_av_frame_frag_t* data, bool isChunkFragment) { uint64_t ts = data->rtp_time; @@ -140,8 +141,7 @@ namespace DepLibSfuShm { && (LastSeq(ShmChunkType::VIDEO) == UINT64_UNSET || data->first_rtp_seq - 1 == LastSeq(ShmChunkType::VIDEO))) { - MS_DEBUG_TAG(rtp, "SHM_Q_PKT_CANWRITETHRU seqid=%" PRIu64 "(%" PRIu64 ") ts=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", - data->first_rtp_seq, LastSeq(ShmChunkType::VIDEO), data->rtp_time, LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); + //MS_DEBUG_TAG(rtp, "WRITETHRU [seq=%" PRIu64 " ts=%" PRIu64 "] len=%zu chunkstart=%s chunkend=%s qsize=%zu", data->first_rtp_seq, data->rtp_time, data->len, isChunkStart? "1":"0", isChunkEnd? "1":"0", videoPktBuffer.size()); return ShmQueueStatus::SHM_Q_PKT_CANWRITETHRU; } @@ -150,8 +150,7 @@ namespace DepLibSfuShm { && LastTs(ShmChunkType::VIDEO) > ts && LastTs(ShmChunkType::VIDEO) - ts > MaxVideoPktDelay) { - MS_DEBUG_TAG(rtp, "SHM_Q_PKT_TOO_OLD seqid=%" PRIu64 " ts=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", - data->first_rtp_seq, data->rtp_time, LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); + MS_DEBUG_TAG(rtp, "REJECT OLD seqid=%" PRIu64 " delta=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", data->first_rtp_seq, LastTs(ShmChunkType::VIDEO) - data->rtp_time, LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); return ShmQueueStatus::SHM_Q_PKT_TOO_OLD; } @@ -163,10 +162,6 @@ namespace DepLibSfuShm { ++it; } it = videoPktBuffer.emplace(it, data, isChunkFragment, isChunkStart, isChunkEnd); - - MS_DEBUG_TAG(rtp, "Successully enqueued data seqId=%" PRIu64 "%s%s%s len=%zu ts=%" PRIu64 " lastSeq=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", - seq, isChunkFragment ? " isFragment" : "", isChunkStart ? " chunkStart" : "", isChunkEnd ? " chunkEnd" : "", it->chunk.len, - ts, LastSeq(ShmChunkType::VIDEO), LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); return SHM_Q_PKT_QUEUED_OK; } @@ -174,13 +169,15 @@ namespace DepLibSfuShm { ShmQueueStatus SfuShmCtx::Dequeue() { - ShmQueueStatus ret = SHM_Q_PKT_DEQUEUED_NOTHING; + ShmQueueStatus ret; int err; if (this->videoPktBuffer.empty()) return SHM_Q_PKT_DEQUEUED_NOTHING; - // TODO: this algorithm tracks previous seq id in a queue, confirm that it is always monotonous + // seqId: strictly +1 unless NALUs came from STAP-A packet, then they are the same + // timestamps: should increment for each new picture (shm chunk), otherwise they match + // Can see two fragments coming in with the same seqId and timestamps: SPS and PPS from the same STAP-A packet are typical std::_List_iterator chunkStartIt; auto it = this->videoPktBuffer.begin(); @@ -190,12 +187,11 @@ namespace DepLibSfuShm { while (it != this->videoPktBuffer.end()) { // First, drop chunks with expired timestamps: last timestamp of a written or skipped packet - if (LastSeq(ShmChunkType::VIDEO) != UINT64_UNSET + if (LastTs(ShmChunkType::VIDEO) != UINT64_UNSET && LastTs(ShmChunkType::VIDEO) > it->chunk.rtp_time - && LastTs(ShmChunkType::VIDEO) - it->chunk.rtp_time > MaxVideoPktDelay) // TODO: review ts check and verify that LastTs() is updated properly in each scenario + && LastTs(ShmChunkType::VIDEO) - it->chunk.rtp_time > MaxVideoPktDelay) { - MS_DEBUG_TAG(rtp, "dropping pkt seqId=%" PRIu64 " ts=%" PRIu64 " LastTs=%" PRIu64 " qsize=%zu", - it->chunk.first_rtp_seq, it->chunk.rtp_time, LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); + MS_DEBUG_TAG(rtp, "DROP OLD [seq=%" PRIu64 " delta=%" PRIu64 "] lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, LastTs(ShmChunkType::VIDEO) - it->chunk.rtp_time, LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); prev_seq = it->chunk.first_rtp_seq; it = this->videoPktBuffer.erase(it); continue; @@ -204,7 +200,7 @@ namespace DepLibSfuShm { // Hole in seqIds, exit and wait for missing pkt to be retransmitted if (it->chunk.first_rtp_seq > prev_seq + 1) { - MS_DEBUG_TAG(rtp, "HOLE in seqId: cur=%" PRIu64 " prev=%" PRIu64, it->chunk.first_rtp_seq, prev_seq); + //MS_DEBUG_TAG(rtp, "HOLE [seq=%" PRIu64 " ts=%" PRIu64 "] prev=%" PRIu64, it->chunk.first_rtp_seq, it->chunk.rtp_time, prev_seq); return SHM_Q_PKT_WAIT_FOR_NACK; } @@ -223,8 +219,7 @@ namespace DepLibSfuShm { if (!chunkStartFound) { // chunk incomplete, wait for retransmission - MS_DEBUG_TAG(rtp, "found fragmented chunk but no start yet: [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", - it->chunk.first_rtp_seq, it->chunk.rtp_time, videoPktBuffer.size()); + // MS_DEBUG_TAG(rtp, "NO START fragment, wait: [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", it->chunk.first_rtp_seq, it->chunk.rtp_time, videoPktBuffer.size()); return SHM_Q_PKT_CHUNKSTART_MISSING; } @@ -241,22 +236,20 @@ namespace DepLibSfuShm { nextit++; // position past last fragment in a chunk while (chunkStartIt != nextit) { - MS_DEBUG_TAG(rtp, "writing fragment seq=%" PRIu64 " [%X%X%X%X]", + /*MS_DEBUG_TAG(rtp, "writing fragment seq=%" PRIu64 " [%X%X%X%X]", chunkStartIt->chunk.first_rtp_seq, - chunkStartIt->chunk.data[0],chunkStartIt->chunk.data[1],chunkStartIt->chunk.data[2],chunkStartIt->chunk.data[3]); + chunkStartIt->chunk.data[0],chunkStartIt->chunk.data[1],chunkStartIt->chunk.data[2],chunkStartIt->chunk.data[3]); */ err = this->WriteChunk(&chunkStartIt->chunk, ShmChunkType::VIDEO); ret = IsError(err) ? SHM_Q_PKT_SHMWRITE_ERR : SHM_Q_PKT_DEQUEUED_OK; chunkStartIt = this->videoPktBuffer.erase(chunkStartIt); } - MS_DEBUG_TAG(rtp, "wrote fragmented chunk! start: [seq=%" PRIu64 " ts=%" PRIu64 "] end: [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", - startSeqId, startTs, endSeqId, endTs, videoPktBuffer.size()); + //MS_DEBUG_TAG(rtp, "WROTE FRAGMENTS start [seq=%" PRIu64 " ts=%" PRIu64 "] end [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", startSeqId, startTs, endSeqId, endTs, videoPktBuffer.size()); chunkStartFound = false; + prev_seq = endSeqId; it = nextit; // restore iterator } else { - MS_DEBUG_TAG(rtp, "skip over mid fragment [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", - it->chunk.first_rtp_seq, it->chunk.rtp_time, videoPktBuffer.size()); prev_seq = it->chunk.first_rtp_seq; ++it; } @@ -265,9 +258,7 @@ namespace DepLibSfuShm { else // non-fragmented chunk { err = this->WriteChunk(&it->chunk, ShmChunkType::VIDEO); - MS_DEBUG_TAG(rtp, "wrote non-fragmented chunk! seqId=%" PRIu64 " ts=%" PRIu64 " qsize=%zu [%X%X%X%X]", - it->chunk.first_rtp_seq, it->chunk.rtp_time, videoPktBuffer.size(), - it->chunk.data[0],it->chunk.data[1],it->chunk.data[2],it->chunk.data[3]); +// MS_DEBUG_TAG(rtp, "WROTE WHOLE [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu [%X%X%X%X]", it->chunk.first_rtp_seq, it->chunk.rtp_time, videoPktBuffer.size(), it->chunk.data[0],it->chunk.data[1],it->chunk.data[2],it->chunk.data[3]); ret = IsError(err) ? SHM_Q_PKT_SHMWRITE_ERR : SHM_Q_PKT_DEQUEUED_OK; prev_seq = it->chunk.first_rtp_seq; it = this->videoPktBuffer.erase(it); @@ -280,7 +271,7 @@ namespace DepLibSfuShm { int SfuShmCtx::WriteRtpDataToShm(ShmChunkType type, sfushm_av_frame_frag_t* frag, bool isChunkFragment) { - int err; + int err = 0; ShmQueueStatus st; if (type == ShmChunkType::VIDEO) @@ -288,9 +279,6 @@ namespace DepLibSfuShm { st = this->Enqueue(frag, isChunkFragment); if (SHM_Q_PKT_CANWRITETHRU == st) // Write it into shm, no need to queue anything { - MS_DEBUG_TAG(rtp, "wrote whole chunk seqid=%" PRIu64 " ts=%" PRIu64 " [%X%X%X%X]", - frag->first_rtp_seq, frag->rtp_time, frag->data[0],frag->data[1],frag->data[2],frag->data[3]); - return this->WriteChunk(frag, ShmChunkType::VIDEO); } @@ -300,9 +288,8 @@ namespace DepLibSfuShm { } st = this->Dequeue(); - if (SHM_Q_PKT_DEQUEUED_OK != st) + if (SHM_Q_PKT_DEQUEUED_OK != st && SHM_Q_PKT_DEQUEUED_NOTHING != st) { - MS_DEBUG_TAG(rtp,"Dequeue() returned state %d", st); return (SHM_Q_PKT_SHMWRITE_ERR == st) ? -1 : 0; } } @@ -317,13 +304,12 @@ namespace DepLibSfuShm { int SfuShmCtx::WriteChunk(sfushm_av_frame_frag_t* data, ShmChunkType kind) { - int err; - if (Status() != SHM_WRT_READY) { return 0; } + int err; switch (kind) { case ShmChunkType::VIDEO: diff --git a/worker/src/RTC/PipeConsumer.cpp b/worker/src/RTC/PipeConsumer.cpp index fce1b0b2f2..109a7771ba 100644 --- a/worker/src/RTC/PipeConsumer.cpp +++ b/worker/src/RTC/PipeConsumer.cpp @@ -552,7 +552,14 @@ namespace RTC auto* rtxCodec = this->rtpParameters.GetRtxCodecForEncoding(encoding); if (rtxCodec && encoding.hasRtx) + { rtpStream->SetRtx(rtxCodec->payloadType, encoding.rtx.ssrc); + MS_DEBUG_2TAGS("RTX set up"); + } + else + { + MS_DEBUG_2TAGS("RTX is NOT set up"); + } this->rtpStreams.push_back(rtpStream); this->mapMappedSsrcRtpStream[encoding.ssrc] = rtpStream; diff --git a/worker/src/RTC/RtpStreamRecv.cpp b/worker/src/RTC/RtpStreamRecv.cpp index 5006567891..ae52ef9eb8 100644 --- a/worker/src/RTC/RtpStreamRecv.cpp +++ b/worker/src/RTC/RtpStreamRecv.cpp @@ -188,12 +188,13 @@ namespace RTC if (this->params.useNack) { this->nackGenerator.reset(new RTC::NackGenerator(this)); + MS_DEBUG_TAG(rtp,"L@@K RtpStreamRecv::params.useNack is true, NACK feature enabled"); } else { - MS_DEBUG_TAG(rtp,"L@@KL@@KL@@K RtpStreamRecv::params.useNack is false, NACK feature disabled"); + MS_DEBUG_TAG(rtp,"L@@K RtpStreamRecv::params.useNack is false, NACK feature disabled"); } - MS_DEBUG_TAG(rtp, "L@@KL@@KL@@K RtpStreamRecv::ctor() HasRtx is %s", HasRtx() ? "TRUE" : "FALSE"); + MS_DEBUG_TAG(rtp, "L@@K RtpStreamRecv::ctor() HasRtx=%s", HasRtx() ? "TRUE" : "FALSE"); // Run the RTP inactivity periodic timer (unless DTX is enabled). if (!this->params.useDtx) diff --git a/worker/src/RTC/ShmConsumer.cpp b/worker/src/RTC/ShmConsumer.cpp index d55c61744f..22b0ede9f8 100644 --- a/worker/src/RTC/ShmConsumer.cpp +++ b/worker/src/RTC/ShmConsumer.cpp @@ -8,6 +8,7 @@ #include "Channel/Notifier.hpp" #include "RTC/Codecs/Tools.hpp" #include "RTC/RTCP/FeedbackRtpNack.hpp" +#include "RTC/RtpStreamRecv.hpp" namespace RTC @@ -282,6 +283,34 @@ namespace RTC origSeq); } + if (packet->IsKeyFrame()) + { + RTC::RtpStreamRecv* recvStream = dynamic_cast(this->producerRtpStream); + if (nullptr != recvStream) + { + MS_DEBUG_TAG(rtp, "Jitter=%" PRIu32, recvStream->GetJitter()); + } + else + { + MS_DEBUG_TAG(rtp, "no Jitter for me :("); + } + } + +/* + if (packet->IsKeyFrame()) + { + MS_DEBUG_TAG( + rtp, + "L@@K KEYFRAME packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 + " payload:%zu] from original [seq:%" PRIu16 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp(), + packet->GetPayloadLength(), + origSeq + ); + } +*/ // Process the packet. In case of shm writer this is still needed for NACKs if (this->rtpStream->ReceivePacket(packet)) { @@ -325,7 +354,7 @@ namespace RTC } } - // Cannot write into shm until the first RTCP Sender Report received + // Before writing to shm must receive RTCP Sender Report or cannot sync if (!this->srReceived) { /* MS_DEBUG_TAG(rtp, "RTCP SR not received yet, ignoring packet[ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", packet->GetSsrc(), @@ -335,12 +364,12 @@ namespace RTC } // Uncomment for NACK test simulation - if (this->TestNACK(packet)) +/* if (this->TestNACK(packet)) { MS_DEBUG_TAG(rtp, "Pretend NACK for packet ssrc:%" PRIu32 ", seq:%" PRIu16 " ts: %" PRIu32 " and wait for retransmission", packet->GetSsrc(), packet->GetSequenceNumber(), packet->GetTimestamp()); return; - } + }*/ // End of NACK test simulation @@ -448,7 +477,9 @@ namespace RTC uint64_t ts = static_cast(packet->GetTimestamp()); uint64_t seq = static_cast(packet->GetSequenceNumber()); uint32_t ssrc = packet->GetSsrc(); - bool isKeyFrame = packet->IsKeyFrame(); //uncomment for debugging output + bool isKeyFrame = packet->IsKeyFrame(); + + int err = 0; std::memset(&chunk, 0, sizeof(chunk)); @@ -478,10 +509,8 @@ namespace RTC { uint8_t nal = *(data) & 0x1F; uint8_t marker = *(pktdata + 1) & 0x80; // Marker bit indicates the last or the only NALU in this packet is the end of the picture data - - // If the first video pkt, or pkt's timestamp is different from the previous video pkt written out - int begin_picture = (shmCtx->IsSeqUnset(DepLibSfuShm::ShmChunkType::VIDEO)) || (ts > shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO)); - + bool begin_picture = (shmCtx->IsSeqUnset(DepLibSfuShm::ShmChunkType::VIDEO) // assume that first video pkt starts the picture frame + || (ts > shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO))); // Single NAL unit packet if (nal >= 1 && nal <= 23) { @@ -516,23 +545,24 @@ namespace RTC this->chunk.ssrc = ssrc; this->chunk.begin = begin_picture; this->chunk.end = (marker != 0); - /* - if (nal != 1) - { - MS_DEBUG_TAG(rtp, "video single NALU=%d LEN=%zu ts %" PRIu64 " seq %" PRIu64 " isKeyFrame=%d begin_picture(chunk.begin)=%d marker(chunk.end)=%d lastTs=%" PRIu64, - nal, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, isKeyFrame, begin_picture, marker, shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO)); - } */ - if (0 != shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &chunk)) + + // if (nal != 1) + // { + // MS_DEBUG_TAG(rtp, "video single NALU=%d LEN=%zu ts %" PRIu64 " seq %" PRIu64 " isKeyFrame=%d begin_picture(chunk.begin)=%d marker(chunk.end)=%d lastTs=%" PRIu64, + // nal, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, isKeyFrame, begin_picture, marker, shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO)); + // } + + if (0 != (err = shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &chunk, !(this->chunk.begin && this->chunk.end)))) { - MS_WARN_TAG(rtp, "FAIL writing video NALU=%d: ts %" PRIu64 " seq %" PRIu64, nal, this->chunk.rtp_time, this->chunk.first_rtp_seq); + MS_WARN_TAG(rtp, "FAIL %d writing video NALU=%d: ts %" PRIu64 " seq %" PRIu64, + err, nal, this->chunk.rtp_time, this->chunk.first_rtp_seq); ret=false; break; } - } // Single NALU pkt + } else { switch (nal) { - // Aggregation packet. Contains several NAL units in a single RTP packet // STAP-A. /* @@ -557,7 +587,7 @@ namespace RTC case 24: { size_t offset{ 1 }; // Skip over STAP-A NAL header - + // Iterate NAL units. while (offset < len - 3) // need to be able to read 2 bytes of NALU size { @@ -568,8 +598,8 @@ namespace RTC } offset += 2; // skip over NALU size - // uint8_t subnal = *(data + offset) & 0x1F; // uncomment for debugging output, NAL type - + uint8_t subnal = *(data + offset) & 0x1F; // actual NAL type + //Add Annex B uint16_t chunksize = naluSize; this->chunk.end = (marker != 0) && (offset + chunksize >= len - 3); @@ -595,19 +625,18 @@ namespace RTC this->chunk.data = data + offset; this->chunk.len = chunksize; this->chunk.rtp_time = ts; - this->chunk.first_rtp_seq = this->chunk.last_rtp_seq = seq; // TODO: is it okay to send several NALUs with same seq? + this->chunk.first_rtp_seq = this->chunk.last_rtp_seq = seq; this->chunk.ssrc = ssrc; - /* if (subnal != 1) +/* + MS_DEBUG_TAG(rtp, "video STAP-A: NAL=%d payloadlen=%" PRIu32 " nalulen=%" PRIu16 " chunklen=%" PRIu32 " ts=%" PRIu64 " seq=%" PRIu64 " lastTs=%" PRIu64 " isKeyFrame=%d chunk.begin=%d chunk.end=%d", + subnal, len, naluSize, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, + shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO), isKeyFrame, this->chunk.begin, this->chunk.end); + */ + if (0 != (err = shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &chunk, !(this->chunk.begin && this->chunk.end)))) { - MS_DEBUG_TAG(rtp, "video STAP-A: NAL=%d payloadlen=%" PRIu32 " nalulen=%" PRIu16 " chunklen=%" PRIu32 " ts=%" PRIu64 " seq=%" PRIu64 " lastTs=%" PRIu64 " isKeyFrame=%d chunk.begin=%d chunk.end=%d", - subnal, len, naluSize, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, - shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO), isKeyFrame, this->chunk.begin, this->chunk.end); - } */ - if (0 != shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &chunk)) - { - MS_WARN_TAG(rtp, "FAIL writing STAP-A pkt to shm: len %zu ts %" PRIu64 " seq %" PRIu64, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq); + MS_WARN_TAG(rtp, "FAIL %d writing STAP-A pkt to shm: len %zu ts %" PRIu64 " seq %" PRIu64, + err, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq); ret=false; - break; } offset += chunksize; @@ -641,9 +670,9 @@ namespace RTC uint8_t startBit = *(data + 1) & 0x80; // 1st bit indicates the starting fragment uint8_t endBit = *(data + 1) & 0x40; // 2nd bit indicates the ending fragment uint8_t fuInd = *(data) & 0xE0; // 3 first bytes of FU indicator, use to compose NAL header for the beginning fragment - uint8_t subnal = *(data + 1) & 0x1F; // Last 5 bits in FU header subtype of FU unit, use to compose NAL header for the beginning fragment, also if (subnal == 7 (SPS) && startBit == 128) then we have a key frame + uint8_t subnal = *(data + 1) & 0x1F; // Last 5 bits in FU header subtype of FU unit, use to compose NAL header for the beginning fragment, also if (videoNALU == 7 (SPS) && startBit == 128) then we have a key frame + size_t chunksize = len - 1; // skip over FU indicator - size_t chunksize = len - 1; // skip over FU indicator if (startBit == 128) { // Write AnnexB marker and put NAL header in place of FU header @@ -682,16 +711,18 @@ namespace RTC this->chunk.first_rtp_seq = this->chunk.last_rtp_seq = seq; this->chunk.ssrc = ssrc; this->chunk.end = (endBit && marker) ? 1 : 0; - /* if (subnal != 1) { + /*if (subnal != 1) + { MS_DEBUG_TAG(rtp, "video FU-A NAL=%" PRIu8 " len=%" PRIu32 " ts=%" PRIu64 " prev_ts=%" PRIu64 " seq=%" PRIu64 " isKeyFrame=%d startBit=%" PRIu8 " endBit=%" PRIu8 " marker=%" PRIu8 " chunk.begin=%d chunk.end=%d", subnal, this->chunk.len, this->chunk.rtp_time, shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO), this->chunk.first_rtp_seq, isKeyFrame, startBit, endBit, marker, this->chunk.begin, this->chunk.end); - } */ - if (0 != shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &(this->chunk), true)) + }*/ + + if (0 != (err = shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &(this->chunk), true))) { - MS_WARN_TAG(rtp, "FAIL writing FU-A pkt to shm: len=%zu ts=%" PRIu64 " seq=%" PRIu64 "%s%s", this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, this->chunk.begin ? " begin" : "", this->chunk.end ? " end": ""); + MS_WARN_TAG(rtp, "FAIL %d writing FU-A pkt to shm: len=%zu ts=%" PRIu64 " seq=%" PRIu64 "%s%s", + err, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, this->chunk.begin ? " begin" : "", this->chunk.end ? " end": ""); ret=false; - break; } break; } diff --git a/worker/src/RTC/ShmTransport.cpp b/worker/src/RTC/ShmTransport.cpp index 381567434b..2c5e152f4d 100644 --- a/worker/src/RTC/ShmTransport.cpp +++ b/worker/src/RTC/ShmTransport.cpp @@ -37,7 +37,7 @@ namespace RTC auto jsonShmNameIt = jsonShmIt->find("name"); if (jsonShmNameIt == jsonShmIt->end()) - MS_THROW_TYPE_ERROR("missing shm.name"); + MS_THROW_TYPE_ERROR("missing shm.name in %s", data.dump().c_str()); else if (!jsonShmNameIt->is_string()) MS_THROW_TYPE_ERROR("wrong shm.name (not a string)"); From 96faaeba15985bbbec4685884f74b5b7316b6569 Mon Sep 17 00:00:00 2001 From: Maria Tverdostup Date: Fri, 14 Aug 2020 21:36:07 -0700 Subject: [PATCH 05/11] Major code cleanup, shm consumer listener, fixes in shm queue --- package.json | 2 +- worker/include/DepLibSfuShm.hpp | 248 ++++++++++++--------------- worker/include/RTC/ShmConsumer.hpp | 22 ++- worker/include/RTC/ShmTransport.hpp | 5 +- worker/src/DepLibSfuShm.cpp | 252 ++++++++++++++-------------- worker/src/RTC/ShmConsumer.cpp | 128 ++++++-------- worker/src/RTC/ShmTransport.cpp | 24 +-- worker/src/RTC/Transport.cpp | 2 +- 8 files changed, 300 insertions(+), 383 deletions(-) diff --git a/package.json b/package.json index bbcc80727a..b8cae670aa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@livelyvideo/mediasoup", - "version": "3.5.7-shm73-rtx18", + "version": "3.5.7-shm73-rtx21", "description": "Cutting Edge WebRTC Video Conferencing", "contributors": [ "Iñaki Baz Castillo (https://inakibaz.me)", diff --git a/worker/include/DepLibSfuShm.hpp b/worker/include/DepLibSfuShm.hpp index 395cb3c797..f4fa8a5199 100644 --- a/worker/include/DepLibSfuShm.hpp +++ b/worker/include/DepLibSfuShm.hpp @@ -10,7 +10,6 @@ extern "C" #include #include #include -#include "RTC/ShmConsumer.hpp" #define UINT64_UNSET ((uint64_t)-1) #define MAX_SEQ_DELTA 100 @@ -29,41 +28,46 @@ namespace DepLibSfuShm // TODO: do I need an explicit value for a writer in error state? }; - enum ShmChunkType + enum Media { VIDEO, - AUDIO, - RTCP, - UNDEFINED + AUDIO + }; + + // If Sender Report arrived but shm writer could not be initialized yet, we can save it and write it in as soon as shm writer is initialized + struct MediaState + { + uint64_t last_seq{ UINT64_UNSET }; // last RTP pkt sequence processed by this input + uint64_t last_ts{ UINT64_UNSET }; // the timestamp of the last processed RTP messpktage + uint32_t ssrc{ 0 }; // ssrc of audio chn + bool sr_received{ false }; + bool sr_written{ false }; + uint32_t sr_ntp_msb{ 0 }; + uint32_t sr_ntp_lsb{ 0 }; + uint64_t sr_rtp_tm{ 0 }; }; // Need this to keep a buffer of roughly 2-3 video frames - enum ShmQueueStatus + enum EnqueueResult { - SHM_Q_PKT_CANWRITETHRU = 99, // queue was empty, the pkt represents whole video frame or it is an aggregate, its seqId is (last_seqId+1), won't add pkt to queue - SHM_Q_PKT_TOO_OLD = 100, // timestamp too old, don't queue - SHM_Q_PKT_QUEUED_OK = 101, // added pkt data into the queue - SHM_Q_PKT_DEQUEUED_OK = 102, // wrote some data into shm on dequeueing - SHM_Q_PKT_DEQUEUED_NOTHING = 103, // nothing was written into shm on dequeueing call - SHM_Q_PKT_WAIT_FOR_NACK = 104, // stopped dequeueing because waiting for retransmission - SHM_Q_PKT_CHUNKSTART_MISSING = 105, // start of fragment chunk missing, dropped the whole chunk - SHM_Q_PKT_SHMWRITE_ERR = 106 // shm writer returned some err, check the logs for details + SHM_Q_PKT_CANWRITETHRU, // queue was empty, the pkt represents whole video frame or it is an aggregate, its seqId is (last_seqId+1), won't add pkt to queue + SHM_Q_PKT_QUEUED_OK // added pkt data into the queue }; // Video NALUs queue item struct ShmQueueItem { - sfushm_av_frame_frag_t chunk; // Cloned data representing either a whole NALU or a fragment - uint8_t store[MTU_SIZE]; // Memory to hold the cloned pkt data, won't be larger than MTU (should match RTC::MtuSize) - bool isChunkFragment{ false }; // If this is a picture's fragment (can be whole NALU or NALU's fragment) - bool isChunkStart{ false }; // The first (or only) picture's fragment - bool isChunkEnd{ false }; // Picture's last (or only) fragment + sfushm_av_frame_frag_t chunk; // Cloned data representing either a whole NALU or a fragment + uint8_t store[MTU_SIZE]; // Memory to hold the cloned pkt data, won't be larger than MTU (should match RTC::MtuSize) + bool isChunkFragment{ false }; // If this is a picture's fragment (can be whole NALU or NALU's fragment) + bool isChunkStart{ false }; // The first (or only) picture's fragment + bool isChunkEnd{ false }; // Picture's last (or only) fragment ShmQueueItem(sfushm_av_frame_frag_t* data, bool isfragment, bool isfragmentstart, bool isfragmentend); }; // Contains shm configuration, writer context (if initialized), writer status - class SfuShmCtx + class ShmCtx { public: class Listener @@ -73,191 +77,145 @@ namespace DepLibSfuShm }; public: - SfuShmCtx(): wrt_ctx(nullptr), last_seq_a(UINT64_UNSET), last_ts_a(UINT64_UNSET), last_seq_v(UINT64_UNSET), last_ts_v(UINT64_UNSET), wrt_status(SHM_WRT_UNDEFINED) {} - ~SfuShmCtx(); + ShmCtx(): wrt_ctx(nullptr), wrt_status(SHM_WRT_UNDEFINED) {} + ~ShmCtx(); void InitializeShmWriterCtx(std::string shm, std::string log, int level, int stdio); void CloseShmWriterCtx(); - ShmWriterStatus Status() const { return this->wrt_status; } - bool CanWrite() const { return (ShmWriterStatus::SHM_WRT_READY == this->wrt_status) && this->srReceived; } - void SetListener(DepLibSfuShm::SfuShmCtx::Listener* l) { this->listener = l; } - uint32_t AudioSsrc() const { return (this->wrt_init.conf.channels[0].audio == 1) ? this->wrt_init.conf.channels[0].ssrc : 0; } - uint32_t VideoSsrc() const { return (this->wrt_init.conf.channels[1].video == 1) ? this->wrt_init.conf.channels[1].ssrc : 0; } - - void SetSsrcInShmConf(uint32_t ssrc, ShmChunkType kind); - - uint64_t AdjustPktTs(uint64_t ts, ShmChunkType kind); - uint64_t AdjustPktSeq(uint64_t seq, ShmChunkType kind); - void UpdatePktStat(uint64_t seq, uint64_t ts, ShmChunkType kind); - bool IsSeqUnset(ShmChunkType kind) const; - bool IsTsUnset(ShmChunkType kind) const; - uint64_t LastTs(ShmChunkType kind) const; - uint64_t LastSeq(ShmChunkType kind) const; - - int WriteRtpDataToShm(ShmChunkType type, sfushm_av_frame_frag_t* data, bool isChunkFragment = false); - int WriteRtcpSenderReportTs(uint64_t lastSenderReportNtpMs, uint32_t lastSenderReporTs, ShmChunkType kind); + bool CanWrite() const; + void SetListener(DepLibSfuShm::ShmCtx::Listener* l) { this->listener = l; } + + void ConfigureMediaSsrc(uint32_t ssrc, Media kind); + + uint64_t AdjustVideoPktTs(uint64_t ts); + uint64_t AdjustAudioPktTs(uint64_t ts); + uint64_t AdjustVideoPktSeq(uint64_t seq); + uint64_t AdjustAudioPktSeq(uint64_t seq); + void UpdatePktStat(uint64_t seq, uint64_t ts, Media kind); + + bool IsVideoSeqUnset() const; + uint64_t LastVideoTs() const; + uint64_t LastVideoSeq() const; + + void WriteRtpDataToShm(Media type, sfushm_av_frame_frag_t* data, bool isChunkFragment = false); + void WriteRtcpSenderReportTs(uint64_t lastSenderReportNtpMs, uint32_t lastSenderReporTs, Media kind); int WriteStreamMeta(std::string metadata, std::string shm); - int WriteVideoOrientation(uint16_t rotation); + void WriteVideoOrientation(uint16_t rotation); + + private: + EnqueueResult Enqueue( sfushm_av_frame_frag_t* data, bool isChunkFragment); + void Dequeue(); + + void WriteChunk(sfushm_av_frame_frag_t* data, Media kind); + void WriteSR(Media kind); bool IsError(int err_code); const char* GetErrorString(int err_code); - private: - ShmQueueStatus Enqueue( sfushm_av_frame_frag_t* data, bool isChunkFragment); - ShmQueueStatus Dequeue(); - int WriteChunk(sfushm_av_frame_frag_t* data, ShmChunkType kind = ShmChunkType::UNDEFINED); - public: std::string stream_name; std::string log_name; - sfushm_av_wr_ctx_t *wrt_ctx; - uint64_t last_seq_a; // last RTP pkt sequence processed by this input - uint64_t last_ts_a; // the timestamp of the last processed RTP pkt - uint64_t last_seq_v; // last RTP pkt sequence processed by this input - uint64_t last_ts_v; // the timestamp of the last processed RTP messpktage - uint32_t ssrc_v; // ssrc of audio chn - uint32_t ssrc_a; // ssrc of video chn private: sfushm_av_writer_init_t wrt_init; + sfushm_av_wr_ctx_t *wrt_ctx; // 0 - audio, 1 - video ShmWriterStatus wrt_status; - bool srReceived{ false }; // do not write into shm until SR received + MediaState data[2]; // 0 - audio, 1 - video std::list videoPktBuffer; // Video frames queue: newest items (by seqId) added at the end of queue, oldest are read from the front + Listener *listener{ nullptr }; // needs to be initialized with smth from ShmConsumer so that we can notify it when shm writer is ready. TODO: see if it's worth the effort static std::unordered_map errToString; - - Listener *listener{ nullptr }; // needs to be initialized with smth from ShmConsumer so that we can notify it when shm writer is ready }; // Inline methods - inline uint64_t SfuShmCtx::AdjustPktTs(uint64_t ts, ShmChunkType kind) - { - uint64_t last_ts; - switch(kind) - { - case ShmChunkType::VIDEO: - last_ts = this->last_ts_v; - break; - - case ShmChunkType::AUDIO: - last_ts = this->last_ts_a; - break; - - default: - return ts; // do nothing - } + // Begin writing into shm when shm writer is initialized (only happens after ssrcs for both audio and video are known) + // and both Sender Reports have been received and written into shm + inline bool ShmCtx::CanWrite() const + { + return nullptr != this->wrt_ctx + && ShmWriterStatus::SHM_WRT_READY == this->wrt_status + && this->data[0].sr_written + && this->data[1].sr_written; + } - if (last_ts != UINT64_UNSET) { - sfushm_av_adjust_for_overflow_32_64(last_ts, &ts, MAX_PTS_DELTA); + inline uint64_t ShmCtx::AdjustAudioPktTs(uint64_t ts) + { + if (this->data[0].last_ts != UINT64_UNSET) { + sfushm_av_adjust_for_overflow_32_64(this->data[0].last_ts, &ts, MAX_PTS_DELTA); } return ts; } - inline uint64_t SfuShmCtx::AdjustPktSeq(uint64_t seq, ShmChunkType kind) + inline uint64_t ShmCtx::AdjustAudioPktSeq(uint64_t seq) { - uint64_t last_seq; - switch(kind) - { - case ShmChunkType::VIDEO: - last_seq = this->last_seq_v; - break; - - case ShmChunkType::AUDIO: - last_seq = this->last_seq_a; - break; + if (this->data[0].last_seq != UINT64_UNSET) { + sfushm_av_adjust_for_overflow_16_64(this->data[0].last_seq, &seq, MAX_SEQ_DELTA); + } + return seq; + } - default: - return seq; // do nothing + inline uint64_t ShmCtx::AdjustVideoPktTs(uint64_t ts) + { + if (this->data[1].last_ts != UINT64_UNSET) { + sfushm_av_adjust_for_overflow_32_64(this->data[1].last_ts, &ts, MAX_PTS_DELTA); } + return ts; + } - if (last_seq != UINT64_UNSET) { - sfushm_av_adjust_for_overflow_16_64(last_seq, &seq, MAX_SEQ_DELTA); + inline uint64_t ShmCtx::AdjustVideoPktSeq(uint64_t seq) + { + if (this->data[1].last_seq != UINT64_UNSET) { + sfushm_av_adjust_for_overflow_16_64(this->data[1].last_seq, &seq, MAX_SEQ_DELTA); } return seq; } - - inline void SfuShmCtx::UpdatePktStat(uint64_t seq, uint64_t ts, ShmChunkType kind) + + inline void ShmCtx::UpdatePktStat(uint64_t seq, uint64_t ts, Media kind) { //No checks, just update values since we have just written that pkt's data into shm - switch(kind) + if (kind == Media::AUDIO) { - case ShmChunkType::VIDEO: - this->last_seq_v = seq; - this->last_ts_v = ts; - break; - - case ShmChunkType::AUDIO: - this->last_seq_a = seq; - this->last_ts_a = ts; - break; - - default: - break; // TODO: will need stats on other pkt types + this->data[0].last_seq = seq; + this->data[0].last_ts = ts; + } + else // video + { + this->data[1].last_seq = seq; + this->data[1].last_ts = ts; } } - - inline bool SfuShmCtx::IsSeqUnset(ShmChunkType kind) const - { - if (kind == ShmChunkType::VIDEO) - return (this->last_seq_v == UINT64_UNSET); - else if (kind == ShmChunkType::AUDIO) - return (this->last_seq_a == UINT64_UNSET); - else - return true; // no support for other pkt types yet - } - - - inline bool SfuShmCtx::IsTsUnset(ShmChunkType kind) const + inline bool ShmCtx::IsVideoSeqUnset() const { - if (kind == ShmChunkType::VIDEO) - return (this->last_ts_v == UINT64_UNSET); - else if (kind == ShmChunkType::AUDIO) - return (this->last_ts_a == UINT64_UNSET); - else - return true; // no support for other pkt types yet + return (this->data[1].last_seq == UINT64_UNSET); } - - inline uint64_t SfuShmCtx::LastTs(ShmChunkType kind) const + inline uint64_t ShmCtx::LastVideoTs() const { - if (kind == ShmChunkType::VIDEO) - return this->last_ts_v; - else if (kind == ShmChunkType::AUDIO) - return this->last_ts_a; - else - return UINT64_UNSET; // no support for other pkt types yet + return this->data[1].last_ts; } - - inline uint64_t SfuShmCtx::LastSeq(ShmChunkType kind) const + inline uint64_t ShmCtx::LastVideoSeq() const { - if (kind == ShmChunkType::VIDEO) - return this->last_seq_v; - else if (kind == ShmChunkType::AUDIO) - return this->last_seq_a; - else - return UINT64_UNSET; // no support for other pkt types yet + return this->data[1].last_seq; } - - inline bool SfuShmCtx::IsError(int err_code) + inline bool ShmCtx::IsError(int err_code) { return (err_code != SFUSHM_AV_OK && err_code != SFUSHM_AV_AGAIN); } - - inline const char* SfuShmCtx::GetErrorString(int err_code) + inline const char* ShmCtx::GetErrorString(int err_code) { static const char* unknownErr = "unknown SFUSHM error"; - auto it = SfuShmCtx::errToString.find(err_code); + auto it = ShmCtx::errToString.find(err_code); - if (it == SfuShmCtx::errToString.end()) + if (it == ShmCtx::errToString.end()) return unknownErr; return it->second; diff --git a/worker/include/RTC/ShmConsumer.hpp b/worker/include/RTC/ShmConsumer.hpp index 2650cc71b7..b4ed9b0df6 100644 --- a/worker/include/RTC/ShmConsumer.hpp +++ b/worker/include/RTC/ShmConsumer.hpp @@ -41,10 +41,10 @@ namespace RTC uint64_t lastSeqId{ 0u }; }; - class ShmConsumer : public RTC::Consumer, public RTC::RtpStreamSend::Listener, public DepLibSfuShm::SfuShmCtx::Listener + class ShmConsumer : public RTC::Consumer, public RTC::RtpStreamSend::Listener, public DepLibSfuShm::ShmCtx::Listener { public: - ShmConsumer(const std::string& id, const std::string& producerId, RTC::Consumer::Listener* listener, json& data, DepLibSfuShm::SfuShmCtx *shmCtx); + ShmConsumer(const std::string& id, const std::string& producerId, RTC::Consumer::Listener* listener, json& data, DepLibSfuShm::ShmCtx *shmCtx); ~ShmConsumer() override; public: @@ -82,7 +82,7 @@ namespace RTC void RequestKeyFrame(); void FillShmWriterStats(json& jsonObject) const; - bool WritePacketToShm(RTC::RtpPacket* packet); + void WritePacketToShm(RTC::RtpPacket* packet); bool VideoOrientationChanged(RTC::RtpPacket* packet); //Uncomment for NACK test simulation bool TestNACK(RTC::RtpPacket* packet); @@ -93,7 +93,7 @@ namespace RTC void OnRtpStreamScore(RTC::RtpStream* rtpStream, uint8_t score, uint8_t previousScore) override; void OnRtpStreamRetransmitRtpPacket(RTC::RtpStreamSend* rtpStream, RTC::RtpPacket* packet) override; - /* Pure virtual methods inherited from DepLibSfuShm::SfuShmCtx::Listener. */ + /* Pure virtual methods inherited from DepLibSfuShm::ShmCtx::Listener. */ public: void OnShmWriterReady() override; @@ -107,14 +107,12 @@ namespace RTC bool syncRequired{ false }; RTC::SeqManager rtpSeqManager; - // Shm writing: a consumer will "send" RTP packets (either audio or video) into shm - // RTCP packets and "app metadata" will be "sent" into shm by ShmTransport object - DepLibSfuShm::SfuShmCtx *shmCtx{ nullptr }; // Handle to shm context which will be received from ShmTransport during transport.consume() - sfushm_av_frame_frag_t chunk; // Structure holding current chunk being written into shm, convenient to reuse timestamps data sometimes - uint16_t rotation{ 0 }; // Current rotation value for video read from RTP packet's videoOrientationExtensionId - bool rotationDetected{ false }; // Whether video rotation data was ever picked in this stream, then we only write it into shm if there was a change - RTC::RtpDataCounter shmWriterCounter; // Use to collect and report shm writing stats, for RTP only (RTCP is not handled by ShmConsumer) TODO: move into ShmCtx - RTC::RtpLostPktRateCounter lostPktRateCounter; + DepLibSfuShm::ShmCtx *shmCtx{ nullptr }; // Handle to shm context which will be received from ShmTransport during transport.consume() + sfushm_av_frame_frag_t chunk; // Structure holding current chunk being written into shm + uint16_t rotation{ 0 }; // Current rotation value for video read from RTP packet's videoOrientationExtensionId + bool rotationDetected{ false }; // Whether video rotation data was ever picked in this stream, then we only write it into shm if there was a change + RTC::RtpDataCounter shmWriterCounter; // Use to collect and report shm writing stats, for RTP only (RTCP is not handled by ShmConsumer) TODO: move into ShmCtx + RTC::RtpLostPktRateCounter lostPktRateCounter; }; /* Inline methods. */ diff --git a/worker/include/RTC/ShmTransport.hpp b/worker/include/RTC/ShmTransport.hpp index 5d88ddf56c..7ec498c33f 100644 --- a/worker/include/RTC/ShmTransport.hpp +++ b/worker/include/RTC/ShmTransport.hpp @@ -31,11 +31,10 @@ namespace RTC void FillJsonStats(json& jsonArray) override; void HandleRequest(Channel::Request* request) override; bool RecvStreamMeta(json& data) override; - DepLibSfuShm::SfuShmCtx* ShmCtx() { return &this->shmCtx; } + DepLibSfuShm::ShmCtx* ShmCtx() { return &this->shmCtx; } private: bool IsConnected() const override; - bool IsFullyConnected() const; void SendRtpPacket(RTC::RtpPacket* packet, RTC::Transport::onSendCallback* cb = nullptr) override; void SendRtcpPacket(RTC::RTCP::Packet* packet) override; void SendSctpData(const uint8_t* data, size_t len) override; @@ -63,7 +62,7 @@ namespace RTC bool comedia{ false }; bool multiSource{ false }; - DepLibSfuShm::SfuShmCtx shmCtx; // shm writer context, needed here to begin shm initialization and correctly report transport stats + DepLibSfuShm::ShmCtx shmCtx; // shm writer context, needed here to begin shm initialization and correctly report transport stats }; } // namespace RTC diff --git a/worker/src/DepLibSfuShm.cpp b/worker/src/DepLibSfuShm.cpp index e6cd2f0013..c298763104 100644 --- a/worker/src/DepLibSfuShm.cpp +++ b/worker/src/DepLibSfuShm.cpp @@ -11,7 +11,7 @@ namespace DepLibSfuShm { - std::unordered_map DepLibSfuShm::SfuShmCtx::errToString = + std::unordered_map ShmCtx::errToString = { { 0, "success (SFUSHM_AV_OK)" }, { -1, "error (SFUSHM_AV_ERR)" }, @@ -22,7 +22,7 @@ namespace DepLibSfuShm { static constexpr uint64_t MaxVideoPktDelay{ 9000 }; // 100ms in samples (90000 sample rate) TODO: redo to hold constant number of picture frames instead? - SfuShmCtx::~SfuShmCtx() + ShmCtx::~ShmCtx() { // Call if writer is not closed if (SHM_WRT_CLOSED != wrt_status && SHM_WRT_UNDEFINED != wrt_status) @@ -32,7 +32,7 @@ namespace DepLibSfuShm { } - void SfuShmCtx::CloseShmWriterCtx() + void ShmCtx::CloseShmWriterCtx() { // Call if writer is not closed if (SHM_WRT_CLOSED != wrt_status) @@ -45,41 +45,52 @@ namespace DepLibSfuShm { } - void SfuShmCtx::SetSsrcInShmConf(uint32_t ssrc, DepLibSfuShm::ShmChunkType kind) + void ShmCtx::ConfigureMediaSsrc(uint32_t ssrc, Media kind) { //Assuming that ssrc does not change, shm writer is initialized, nothing else to do - if (SHM_WRT_READY == this->Status()) + if (SHM_WRT_READY == this->wrt_status) return; - switch(kind) { - case DepLibSfuShm::ShmChunkType::AUDIO: - ssrc_a = ssrc; - this->wrt_init.conf.channels[0].audio = 1; - this->wrt_init.conf.channels[0].ssrc = ssrc; - this->wrt_status = (this->wrt_status == SHM_WRT_AUDIO_CHNL_CONF_MISSING) ? SHM_WRT_READY : SHM_WRT_VIDEO_CHNL_CONF_MISSING; - break; - - case DepLibSfuShm::ShmChunkType::VIDEO: - ssrc_v = ssrc; - this->wrt_init.conf.channels[1].video = 1; - this->wrt_init.conf.channels[1].ssrc = ssrc; - this->wrt_status = (this->wrt_status == SHM_WRT_VIDEO_CHNL_CONF_MISSING) ? SHM_WRT_READY : SHM_WRT_AUDIO_CHNL_CONF_MISSING; - break; - - default: - return; + if(kind == Media::AUDIO) + { + this->data[0].ssrc = ssrc; + this->wrt_init.conf.channels[0].audio = 1; + this->wrt_init.conf.channels[0].ssrc = ssrc; + this->wrt_status = (this->wrt_status == SHM_WRT_AUDIO_CHNL_CONF_MISSING) ? SHM_WRT_READY : SHM_WRT_VIDEO_CHNL_CONF_MISSING; + } + else // video + { + this->data[1].ssrc = ssrc; + this->wrt_init.conf.channels[1].video = 1; + this->wrt_init.conf.channels[1].ssrc = ssrc; + this->wrt_status = (this->wrt_status == SHM_WRT_VIDEO_CHNL_CONF_MISSING) ? SHM_WRT_READY : SHM_WRT_AUDIO_CHNL_CONF_MISSING; } if (this->wrt_status == SHM_WRT_READY) { + // Just switched into ready status, can open a writer int err = SFUSHM_AV_OK; if ((err = sfushm_av_open_writer( &wrt_init, &wrt_ctx)) != SFUSHM_AV_OK) { MS_DEBUG_TAG(rtp, "FAILED in sfushm_av_open_writer() to initialize sfu shm %s with error %s", this->wrt_init.stream_name, GetErrorString(err)); - wrt_status = SHM_WRT_UNDEFINED; + this->wrt_status = SHM_WRT_UNDEFINED; } else - { - if (this->srReceived) + { + // Write any stored sender reports into shm and possibly notify ShmConsumer + if (this->data[0].sr_received) + { + WriteSR(Media::AUDIO); + this->data[0].sr_written = true; + } + + if (this->data[1].sr_received) + { + WriteSR(Media::VIDEO); + this->data[1].sr_written = true; + } + + if (this->CanWrite()) { + MS_DEBUG_TAG(rtp, "shm is ready and first SRs received and written"); this->listener->OnShmWriterReady(); } } @@ -87,7 +98,7 @@ namespace DepLibSfuShm { } - void SfuShmCtx::InitializeShmWriterCtx(std::string shm, std::string log, int level, int stdio) + void ShmCtx::InitializeShmWriterCtx(std::string shm, std::string log, int level, int stdio) { MS_TRACE(); @@ -133,7 +144,7 @@ namespace DepLibSfuShm { // seqId: strictly +1 unless NALUs came from STAP-A packet, then they are the same // timestamps: should increment for each new picture (shm chunk), otherwise they match // Can see two fragments coming in with the same seqId and timestamps: SPS and PPS from the same STAP-A packet are typical - ShmQueueStatus SfuShmCtx::Enqueue(sfushm_av_frame_frag_t* data, bool isChunkFragment) + EnqueueResult ShmCtx::Enqueue(sfushm_av_frame_frag_t* data, bool isChunkFragment) { uint64_t ts = data->rtp_time; uint64_t seq = data->first_rtp_seq; @@ -143,20 +154,19 @@ namespace DepLibSfuShm { // If queue is empty, seqid is in order, and pkt is not a fragment, there is no need to enqueue if (this->videoPktBuffer.empty() && !isChunkFragment - && (LastSeq(ShmChunkType::VIDEO) == UINT64_UNSET - || data->first_rtp_seq - 1 == LastSeq(ShmChunkType::VIDEO))) + && (IsVideoSeqUnset() + || data->first_rtp_seq - 1 == LastVideoSeq())) { //MS_DEBUG_TAG(rtp, "WRITETHRU [seq=%" PRIu64 " ts=%" PRIu64 "] len=%zu chunkstart=%s chunkend=%s qsize=%zu", data->first_rtp_seq, data->rtp_time, data->len, isChunkStart? "1":"0", isChunkEnd? "1":"0", videoPktBuffer.size()); - return ShmQueueStatus::SHM_Q_PKT_CANWRITETHRU; + return EnqueueResult::SHM_Q_PKT_CANWRITETHRU; } - // Try adding too old pkt in queue anyway - if (LastSeq(ShmChunkType::VIDEO) != UINT64_UNSET - && LastTs(ShmChunkType::VIDEO) > ts - && LastTs(ShmChunkType::VIDEO) - ts > MaxVideoPktDelay) + // Add a too old pkt in queue anyway + if (!IsVideoSeqUnset() + && LastVideoTs() > ts + && LastVideoTs() - ts > MaxVideoPktDelay) { - MS_DEBUG_TAG(rtp, "ENQUEUE OLD seqid=%" PRIu64 " delta=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", data->first_rtp_seq, LastTs(ShmChunkType::VIDEO) - data->rtp_time, LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); -// return ShmQueueStatus::SHM_Q_PKT_TOO_OLD; + MS_DEBUG_TAG(rtp, "ENQUEUE OLD seqid=%" PRIu64 " delta=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", data->first_rtp_seq, LastVideoTs() - data->rtp_time, LastVideoTs(), videoPktBuffer.size()); } // Enqueue pkt, newest at the end @@ -172,17 +182,15 @@ namespace DepLibSfuShm { } - ShmQueueStatus SfuShmCtx::Dequeue() + void ShmCtx::Dequeue() { - ShmQueueStatus ret = SHM_Q_PKT_DEQUEUED_OK; - if (this->videoPktBuffer.empty()) - return SHM_Q_PKT_DEQUEUED_NOTHING; + return; // seqId: strictly +1 unless NALUs came from STAP-A packet, then they are the same // timestamps: should increment for each new picture (shm chunk), otherwise they match // Can see two fragments coming in with the same seqId and timestamps: SPS and PPS from the same STAP-A packet are typical - std::_List_iterator chunkStartIt; + std::_List_iterator chunkStartIt; auto it = this->videoPktBuffer.begin(); uint64_t prev_seq = it->chunk.first_rtp_seq - 1; @@ -191,14 +199,13 @@ namespace DepLibSfuShm { while (it != this->videoPktBuffer.end()) { // First, write out all chunks with expired timestamps. TODO: mark incomplete fragmented pictures as corrupted (feature TBD in shm writer) - // Note that this code path will write into shm all old frames (or their parts) regardless of whether there was a hole between seqIds. - if (LastTs(ShmChunkType::VIDEO) != UINT64_UNSET - && LastTs(ShmChunkType::VIDEO) > it->chunk.rtp_time - && LastTs(ShmChunkType::VIDEO) - it->chunk.rtp_time > MaxVideoPktDelay) + if (!IsVideoSeqUnset() + && LastVideoTs() > it->chunk.rtp_time + && LastVideoTs() - it->chunk.rtp_time > MaxVideoPktDelay) { - MS_DEBUG_TAG(rtp, "WRITE OLD [seq=%" PRIu64 " delta=%" PRIu64 "] lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, LastTs(ShmChunkType::VIDEO) - it->chunk.rtp_time, LastTs(ShmChunkType::VIDEO), videoPktBuffer.size()); + MS_DEBUG_TAG(rtp, "OLD [seq=%" PRIu64 " delta=%" PRIu64 "] lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, LastVideoTs() - it->chunk.rtp_time, LastVideoTs(), videoPktBuffer.size()); prev_seq = it->chunk.first_rtp_seq; - this->WriteChunk(&it->chunk, ShmChunkType::VIDEO); + this->WriteChunk(&it->chunk, Media::VIDEO); it = this->videoPktBuffer.erase(it); continue; } @@ -207,7 +214,7 @@ namespace DepLibSfuShm { if (it->chunk.first_rtp_seq > prev_seq + 1) { MS_DEBUG_TAG(rtp, "HOLE [seq=%" PRIu64 " ts=%" PRIu64 "] prev=%" PRIu64, it->chunk.first_rtp_seq, it->chunk.rtp_time, prev_seq); - return SHM_Q_PKT_WAIT_FOR_NACK; + return; } // Fragment of a chunk: start, end or middle @@ -226,7 +233,7 @@ namespace DepLibSfuShm { { // chunk incomplete, wait for retransmission // MS_DEBUG_TAG(rtp, "NO START fragment, wait: [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", it->chunk.first_rtp_seq, it->chunk.rtp_time, videoPktBuffer.size()); - return SHM_Q_PKT_CHUNKSTART_MISSING; + return; } // write the whole chunk if we have it @@ -245,10 +252,10 @@ namespace DepLibSfuShm { /*MS_DEBUG_TAG(rtp, "writing fragment seq=%" PRIu64 " [%X%X%X%X]", chunkStartIt->chunk.first_rtp_seq, chunkStartIt->chunk.data[0],chunkStartIt->chunk.data[1],chunkStartIt->chunk.data[2],chunkStartIt->chunk.data[3]); */ - this->WriteChunk(&chunkStartIt->chunk, ShmChunkType::VIDEO); + this->WriteChunk(&chunkStartIt->chunk, Media::VIDEO); chunkStartIt = this->videoPktBuffer.erase(chunkStartIt); } - MS_DEBUG_TAG(rtp, "WROTE FRAGMENTS start [seq=%" PRIu64 " ts=%" PRIu64 "] end [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", startSeqId, startTs, endSeqId, endTs, videoPktBuffer.size()); + MS_DEBUG_TAG(rtp, "FRAG start [seq=%" PRIu64 " ts=%" PRIu64 "] end [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", startSeqId, startTs, endSeqId, endTs, videoPktBuffer.size()); chunkStartFound = false; prev_seq = endSeqId; it = nextit; // restore iterator @@ -262,94 +269,58 @@ namespace DepLibSfuShm { } else // non-fragmented chunk { - this->WriteChunk(&it->chunk, ShmChunkType::VIDEO); + this->WriteChunk(&it->chunk, Media::VIDEO); prev_seq = it->chunk.first_rtp_seq; it = this->videoPktBuffer.erase(it); continue; } } - return ret; } - int SfuShmCtx::WriteRtpDataToShm(ShmChunkType type, sfushm_av_frame_frag_t* frag, bool isChunkFragment) + void ShmCtx::WriteRtpDataToShm(Media type, sfushm_av_frame_frag_t* frag, bool isChunkFragment) { - int err = 0; - ShmQueueStatus st; - - if (type == ShmChunkType::VIDEO) + if (type == Media::VIDEO) { - st = this->Enqueue(frag, isChunkFragment); - if (SHM_Q_PKT_CANWRITETHRU == st) // Write it into shm, no need to queue anything + if (SHM_Q_PKT_CANWRITETHRU == this->Enqueue(frag, isChunkFragment)) // Write it into shm, no need to queue anything { - this->WriteChunk(frag, ShmChunkType::VIDEO); + this->WriteChunk(frag, Media::VIDEO); } - st = this->Dequeue(); + this->Dequeue(); } else { - this->WriteChunk(frag, ShmChunkType::AUDIO); // do not queue audio + this->WriteChunk(frag, Media::AUDIO); // do not queue audio } - - return err; } - int SfuShmCtx::WriteChunk(sfushm_av_frame_frag_t* data, ShmChunkType kind) + void ShmCtx::WriteChunk(sfushm_av_frame_frag_t* data, Media kind) { - if (!this->CanWrite()) //Status() != SHM_WRT_READY) + if (!this->CanWrite()) { - return 0; + MS_WARN_TAG(rtp, "Cannot write chunk ssrc=%" PRIu32 " seq=%" PRIu64 " ts=%" PRIu64 " because shm writer is not initialized"); + return; } int err; - switch (kind) + if(kind == Media::AUDIO) { - case ShmChunkType::VIDEO: - err = sfushm_av_write_video(wrt_ctx, data); - break; - - case ShmChunkType::AUDIO: - err = sfushm_av_write_audio(wrt_ctx, data); - break; - - default: - return -1; + err = sfushm_av_write_audio(wrt_ctx, data); } - - if (IsError(err)) + else { - // TODO: also log first_rtp_seq, ssrc and timestamp - MS_WARN_TAG(rtp, "ERROR writing chunk to shm: %d - %s", err, GetErrorString(err)); - return -1; + err = sfushm_av_write_video(wrt_ctx, data); } - return 0; + + if (IsError(err)) + MS_WARN_TAG(rtp, "ERROR writing chunk ssrc=%" PRIu32 " seq=%" PRIu64 " ts=%" PRIu64 " to shm: %d - %s", data->ssrc, data->first_rtp_seq, data->rtp_time, err, GetErrorString(err)); } - int SfuShmCtx::WriteRtcpSenderReportTs(uint64_t lastSenderReportNtpMs, uint32_t lastSenderReporTs, DepLibSfuShm::ShmChunkType kind) + void ShmCtx::WriteRtcpSenderReportTs(uint64_t lastSenderReportNtpMs, uint32_t lastSenderReporTs, Media kind) { - /* if (Status() != SHM_WRT_READY) - { - return 0; - }*/ - - uint32_t ssrc; - switch(kind) - { - case DepLibSfuShm::ShmChunkType::AUDIO: - ssrc = ssrc_a; - break; - - case DepLibSfuShm::ShmChunkType::VIDEO: - ssrc = ssrc_v; - break; - - default: - return 0; - } - /* .. c:function:: uint64_t uv_hrtime(void) @@ -357,9 +328,6 @@ namespace DepLibSfuShm { nanoseconds. It is relative to an arbitrary time in the past. It is not related to the time of day and therefore not subject to clock drift. The primary use is for measuring performance between intervals. */ - auto ntp = Utils::Time::TimeMs2Ntp(lastSenderReportNtpMs); - auto ntp_sec = ntp.seconds; - auto ntp_frac = ntp.fractions; /* Uncomment for debugging uint64_t walltime = DepLibUV::GetTimeMs(); @@ -379,27 +347,63 @@ namespace DepLibSfuShm { ct, clockTimeMs); */ - int err = sfushm_av_write_rtcp_sr_ts(wrt_ctx, ntp_sec, ntp_frac, lastSenderReporTs, ssrc); + auto ntp = Utils::Time::TimeMs2Ntp(lastSenderReportNtpMs); + size_t idx = (kind == Media::AUDIO) ? 0 : 1; - if (IsError(err)) + this->data[idx].sr_received = true; + this->data[idx].sr_ntp_msb = ntp.seconds; + this->data[idx].sr_ntp_lsb = ntp.fractions; + this->data[idx].sr_rtp_tm = lastSenderReporTs; + + if (SHM_WRT_READY != this->wrt_status) + return; // shm writer is not set up yet + + WriteSR(kind); + + if (this->CanWrite()) + return; // already submitted the first SRs into shm, and it is in ready state + + bool shouldNotify = false; + if(kind == Media::AUDIO) { - MS_WARN_TAG(rtp, "ERROR writing RTCP SR: %d - %s", err, GetErrorString(err)); - return -1; + if (this->data[1].sr_written && !this->data[0].sr_written) + shouldNotify = true; + this->data[0].sr_written = true; + } + else + { + if (this->data[0].sr_written && !this->data[1].sr_written) + shouldNotify = true; + this->data[1].sr_written = true; } - if (!this->srReceived && ShmWriterStatus::SHM_WRT_READY == this->wrt_status) + if (shouldNotify) { + MS_DEBUG_TAG(rtp, "First SRs received, and shm is ready"); this->listener->OnShmWriterReady(); } + } - this->srReceived = true; - return 0; + void ShmCtx::WriteSR(Media kind) + { + size_t idx = (kind == Media::AUDIO) ? 0 : 1; + int err = sfushm_av_write_rtcp_sr_ts(wrt_ctx, this->data[idx].sr_ntp_msb, this->data[idx].sr_ntp_lsb, this->data[idx].sr_rtp_tm, this->data[idx].ssrc); + if (IsError(err)) + { + MS_WARN_TAG(rtp, "ERROR writing RTCP SR for ssrc %" PRIu32 " %d - %s", this->data[idx].ssrc, err, GetErrorString(err)); + } } - int SfuShmCtx::WriteStreamMeta(std::string metadata, std::string shm) + int ShmCtx::WriteStreamMeta(std::string metadata, std::string shm) { + if (SHM_WRT_READY != this->wrt_status) + { + MS_WARN_TAG(rtp, "Cannot write stream metadata because shm writer is not initialized"); + return -1; // shm writer is not set up yet + } + int err; uint8_t data[256]; // Currently this is overkill because just 1 byte will be written successfully into shm @@ -416,7 +420,6 @@ namespace DepLibSfuShm { std::copy(metadata.begin(), metadata.end(), data); err = sfushm_av_write_stream_metadata(wrt_ctx, data, metadata.length()); - if (IsError(err)) { MS_WARN_TAG(rtp, "ERROR writing stream metadata: %d - %s", err, GetErrorString(err)); @@ -427,16 +430,17 @@ namespace DepLibSfuShm { } - int SfuShmCtx::WriteVideoOrientation(uint16_t rotation) + void ShmCtx::WriteVideoOrientation(uint16_t rotation) { + if (SHM_WRT_READY != this->wrt_status) + { + MS_WARN_TAG(rtp, "Cannot write video rotation because shm writer is not initialized"); + return; // shm writer is not set up yet + } + int err = sfushm_av_write_video_rotation(wrt_ctx, rotation); if (IsError(err)) - { MS_WARN_TAG(rtp, "ERROR writing video rotation: %d - %s", err, GetErrorString(err)); - return -1; - } - - return 0; } } // namespace DepLibSfuShm \ No newline at end of file diff --git a/worker/src/RTC/ShmConsumer.cpp b/worker/src/RTC/ShmConsumer.cpp index 87e66e3391..f11dcd6345 100644 --- a/worker/src/RTC/ShmConsumer.cpp +++ b/worker/src/RTC/ShmConsumer.cpp @@ -46,7 +46,7 @@ namespace RTC } - ShmConsumer::ShmConsumer(const std::string& id, const std::string& producerId, RTC::Consumer::Listener* listener, json& data, DepLibSfuShm::SfuShmCtx *shmCtx) + ShmConsumer::ShmConsumer(const std::string& id, const std::string& producerId, RTC::Consumer::Listener* listener, json& data, DepLibSfuShm::ShmCtx *shmCtx) : RTC::Consumer::Consumer(id, producerId, listener, data, RTC::RtpParameters::Type::SHM) { MS_TRACE(); @@ -192,6 +192,7 @@ namespace RTC // Emit the score event. } + void ShmConsumer::ProducerRtcpSenderReport(RTC::RtpStream* /*rtpStream*/, bool /*first*/) { MS_TRACE(); @@ -201,19 +202,14 @@ namespace RTC MS_DEBUG_TAG(rtp, "Producer stream failed to read SR RTCP msg"); return; } - else - { - uint64_t lastSenderReportNtpMs = this->producerRtpStream->GetSenderReportNtpMs(); // NTP timestamp in last Sender Report (in ms) - uint32_t lastSenderReporTs = this->producerRtpStream->GetSenderReportTs(); // RTP timestamp in last Sender Report. - if (0 == shmCtx->WriteRtcpSenderReportTs(lastSenderReportNtpMs, lastSenderReporTs, - (this->GetKind() == RTC::Media::Kind::AUDIO) ? DepLibSfuShm::ShmChunkType::AUDIO : DepLibSfuShm::ShmChunkType::VIDEO)) - { - MS_DEBUG_TAG(rtp, "First SR received and written into shm"); - } - } + uint64_t lastSenderReportNtpMs = this->producerRtpStream->GetSenderReportNtpMs(); // NTP timestamp in last Sender Report (in ms) + uint32_t lastSenderReporTs = this->producerRtpStream->GetSenderReportTs(); // RTP timestamp in last Sender Report. + + shmCtx->WriteRtcpSenderReportTs(lastSenderReportNtpMs, lastSenderReporTs, (this->GetKind() == RTC::Media::Kind::AUDIO) ? DepLibSfuShm::Media::AUDIO : DepLibSfuShm::Media::VIDEO); } + void ShmConsumer::SendRtpPacket(RTC::RtpPacket* packet) { MS_TRACE(); @@ -301,8 +297,8 @@ namespace RTC // Need both audio and video SSRCs to set up shm writer if (shmWriterCounter.GetPacketCount() == 0) { - auto kind = (this->GetKind() == RTC::Media::Kind::AUDIO) ? DepLibSfuShm::ShmChunkType::AUDIO : DepLibSfuShm::ShmChunkType::VIDEO; - shmCtx->SetSsrcInShmConf(packet->GetSsrc(), kind)); + auto kind = (this->GetKind() == RTC::Media::Kind::AUDIO) ? DepLibSfuShm::Media::AUDIO : DepLibSfuShm::Media::VIDEO; + shmCtx->ConfigureMediaSsrc(packet->GetSsrc(), kind); } if (!shmCtx->CanWrite() || ignorePkt) // shm is not ready for writing, or we still need a key frame @@ -342,21 +338,10 @@ namespace RTC // End of NACK test simulation - if (this->WritePacketToShm(packet)) - { - // Increase transmission counter. - this->shmWriterCounter.Update(packet); - } - else - { - MS_WARN_TAG( - rtp, - "failed to write to shm packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "] from original [seq:%" PRIu16 "]", - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp(), - origSeq); - } + this->WritePacketToShm(packet); + + // Increase transmission counter. + this->shmWriterCounter.Update(packet); // Restore packet fields. packet->SetSsrc(origSsrc); @@ -433,11 +418,10 @@ namespace RTC } - bool ShmConsumer::WritePacketToShm(RTC::RtpPacket* packet) + void ShmConsumer::WritePacketToShm(RTC::RtpPacket* packet) { MS_TRACE(); - bool ret = true; uint8_t const* pktdata = packet->GetData(); uint8_t const* cdata = packet->GetPayload(); uint8_t* data = const_cast(cdata); @@ -447,12 +431,18 @@ namespace RTC uint32_t ssrc = packet->GetSsrc(); bool isKeyFrame = packet->IsKeyFrame(); - int err = 0; - std::memset(&chunk, 0, sizeof(chunk)); - ts = shmCtx->AdjustPktTs(ts, this->GetKind() == Media::Kind::VIDEO ? DepLibSfuShm::ShmChunkType::VIDEO : DepLibSfuShm::ShmChunkType::AUDIO); - seq = shmCtx->AdjustPktSeq(seq, this->GetKind() == Media::Kind::VIDEO ? DepLibSfuShm::ShmChunkType::VIDEO : DepLibSfuShm::ShmChunkType::AUDIO); + if (this->GetKind() == Media::Kind::VIDEO) + { + ts = shmCtx->AdjustVideoPktTs(ts); + seq = shmCtx->AdjustVideoPktSeq(seq); + } + else // audio + { + ts = shmCtx->AdjustAudioPktTs(ts); + seq = shmCtx->AdjustAudioPktSeq(seq); + } switch (this->GetKind()) { @@ -465,11 +455,7 @@ namespace RTC this->chunk.first_rtp_seq = this->chunk.last_rtp_seq = seq; this->chunk.ssrc = packet->GetSsrc(); this->chunk.begin = this->chunk.end = 1; - if(0 != shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::AUDIO, &chunk)) - { - MS_WARN_TAG(rtp, "FAIL writing audio ts %" PRIu64 " seq %" PRIu64, this->chunk.rtp_time, this->chunk.first_rtp_seq); - ret = false; - } + shmCtx->WriteRtpDataToShm(DepLibSfuShm::Media::AUDIO, &chunk); break; } // audio @@ -477,8 +463,7 @@ namespace RTC { uint8_t nal = *(data) & 0x1F; uint8_t marker = *(pktdata + 1) & 0x80; // Marker bit indicates the last or the only NALU in this packet is the end of the picture data - bool begin_picture = (shmCtx->IsSeqUnset(DepLibSfuShm::ShmChunkType::VIDEO) // assume that first video pkt starts the picture frame - || (ts > shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO))); + bool begin_picture = (shmCtx->IsVideoSeqUnset() || (ts > shmCtx->LastVideoTs())); // assume that first video pkt starts the picture frame // Single NAL unit packet if (nal >= 1 && nal <= 23) { @@ -514,19 +499,14 @@ namespace RTC this->chunk.begin = begin_picture; this->chunk.end = (marker != 0); - // if (nal != 1) - // { - // MS_DEBUG_TAG(rtp, "video single NALU=%d LEN=%zu ts %" PRIu64 " seq %" PRIu64 " isKeyFrame=%d begin_picture(chunk.begin)=%d marker(chunk.end)=%d lastTs=%" PRIu64, - // nal, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, isKeyFrame, begin_picture, marker, shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO)); - // } - - if (0 != (err = shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &chunk, !(this->chunk.begin && this->chunk.end)))) + //if (nal != 1) + if (isKeyFrame) { - MS_WARN_TAG(rtp, "FAIL %d writing video NALU=%d: ts %" PRIu64 " seq %" PRIu64, - err, nal, this->chunk.rtp_time, this->chunk.first_rtp_seq); - ret=false; - break; - } + MS_DEBUG_TAG(rtp, "video single NALU=%d LEN=%zu ts %" PRIu64 " seq %" PRIu64 " isKeyFrame=%d begin_picture(chunk.begin)=%d marker(chunk.end)=%d lastTs=%" PRIu64, + nal, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, isKeyFrame, begin_picture, marker, shmCtx->LastVideoTs()); + } + + shmCtx->WriteRtpDataToShm(DepLibSfuShm::Media::VIDEO, &chunk, !(this->chunk.begin && this->chunk.end)); } else { switch (nal) @@ -595,18 +575,15 @@ namespace RTC this->chunk.rtp_time = ts; this->chunk.first_rtp_seq = this->chunk.last_rtp_seq = seq; this->chunk.ssrc = ssrc; -/* - MS_DEBUG_TAG(rtp, "video STAP-A: NAL=%d payloadlen=%" PRIu32 " nalulen=%" PRIu16 " chunklen=%" PRIu32 " ts=%" PRIu64 " seq=%" PRIu64 " lastTs=%" PRIu64 " isKeyFrame=%d chunk.begin=%d chunk.end=%d", - subnal, len, naluSize, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, - shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO), isKeyFrame, this->chunk.begin, this->chunk.end); - */ - if (0 != (err = shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &chunk, !(this->chunk.begin && this->chunk.end)))) + + if (isKeyFrame) { - MS_WARN_TAG(rtp, "FAIL %d writing STAP-A pkt to shm: len %zu ts %" PRIu64 " seq %" PRIu64, - err, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq); - ret=false; + MS_DEBUG_TAG(rtp, "video STAP-A: NAL=%d payloadlen=%" PRIu32 " nalulen=%" PRIu16 " chunklen=%" PRIu32 " ts=%" PRIu64 " seq=%" PRIu64 " lastTs=%" PRIu64 " isKeyFrame=%d chunk.begin=%d chunk.end=%d", + subnal, len, naluSize, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, + shmCtx->LastVideoTs(), isKeyFrame, this->chunk.begin, this->chunk.end); } - + + shmCtx->WriteRtpDataToShm(DepLibSfuShm::Media::VIDEO, &chunk, !(this->chunk.begin && this->chunk.end)); offset += chunksize; } break; @@ -631,7 +608,6 @@ namespace RTC if (len < 3) { MS_DEBUG_TAG(rtp, "FU-A payload too short"); - ret=false; break; } // Parse FU header octet @@ -679,19 +655,14 @@ namespace RTC this->chunk.first_rtp_seq = this->chunk.last_rtp_seq = seq; this->chunk.ssrc = ssrc; this->chunk.end = (endBit && marker) ? 1 : 0; - /*if (subnal != 1) + if (isKeyFrame) //(subnal != 1) { MS_DEBUG_TAG(rtp, "video FU-A NAL=%" PRIu8 " len=%" PRIu32 " ts=%" PRIu64 " prev_ts=%" PRIu64 " seq=%" PRIu64 " isKeyFrame=%d startBit=%" PRIu8 " endBit=%" PRIu8 " marker=%" PRIu8 " chunk.begin=%d chunk.end=%d", - subnal, this->chunk.len, this->chunk.rtp_time, shmCtx->LastTs(DepLibSfuShm::ShmChunkType::VIDEO), this->chunk.first_rtp_seq, + subnal, this->chunk.len, this->chunk.rtp_time, shmCtx->LastVideoTs(), this->chunk.first_rtp_seq, isKeyFrame, startBit, endBit, marker, this->chunk.begin, this->chunk.end); - }*/ - - if (0 != (err = shmCtx->WriteRtpDataToShm(DepLibSfuShm::ShmChunkType::VIDEO, &(this->chunk), true))) - { - MS_WARN_TAG(rtp, "FAIL %d writing FU-A pkt to shm: len=%zu ts=%" PRIu64 " seq=%" PRIu64 "%s%s", - err, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, this->chunk.begin ? " begin" : "", this->chunk.end ? " end": ""); - ret=false; } + + shmCtx->WriteRtpDataToShm(DepLibSfuShm::Media::VIDEO, &(this->chunk), true); break; } case 25: // STAB-B @@ -700,13 +671,11 @@ namespace RTC case 29: // FU-B { MS_WARN_TAG(rtp, "Unsupported NAL unit type %u in video packet", nal); - ret=false; break; } default: // ignore the rest { MS_DEBUG_TAG(rtp, "Unknown NAL unit type %u in video packet", nal); - ret=false; break; } } @@ -722,15 +691,14 @@ namespace RTC switch (this->GetKind()) { case RTC::Media::Kind::AUDIO: - shmCtx->UpdatePktStat(seq, ts, DepLibSfuShm::ShmChunkType::AUDIO); + shmCtx->UpdatePktStat(seq, ts, DepLibSfuShm::Media::AUDIO); break; case RTC::Media::Kind::VIDEO: - shmCtx->UpdatePktStat(seq, ts, DepLibSfuShm::ShmChunkType::VIDEO); + shmCtx->UpdatePktStat(seq, ts, DepLibSfuShm::Media::VIDEO); break; default: break; } - return ret; } @@ -955,7 +923,9 @@ namespace RTC void ShmConsumer::OnShmWriterReady() { MS_TRACE(); - + + MS_DEBUG_TAG(rtp, "writer ready, consumer %s, get key frame", IsActive() ? "ready" : "not ready"); + this->syncRequired = true; if (IsActive()) diff --git a/worker/src/RTC/ShmTransport.cpp b/worker/src/RTC/ShmTransport.cpp index d7257faac5..c50630c705 100644 --- a/worker/src/RTC/ShmTransport.cpp +++ b/worker/src/RTC/ShmTransport.cpp @@ -173,7 +173,7 @@ namespace RTC { MS_TRACE(); - if (!IsConnected()) //IsFullyConnected()) + if (!IsConnected()) { return; } @@ -196,7 +196,7 @@ namespace RTC { MS_TRACE(); - if (!IsConnected()) //IsFullyConnected()) + if (!IsConnected()) return; RTC::RTCP::Packet* packet = RTC::RTCP::Packet::Parse(data, len); @@ -217,7 +217,7 @@ namespace RTC { MS_TRACE(); - if (!IsConnected()) //IsFullyConnected()) + if (!IsConnected()) return; // Pass it to the parent transport. @@ -254,29 +254,17 @@ namespace RTC } - /* - Transport's consumer expects 'true' from IsConnected() in order to activate. - ShmTransport::IsConnected() rather means that transport is initialized but it may not be "fully connected" and ready to write into shm yet. - To write into shm we wait until both audio and video consumers receive their first RTP packets with ssrc values. - Call ShmTransport::IsFullyConnected() to detect if ShmTransport is ready to write into shm. - */ inline bool ShmTransport::IsConnected() const { return true; } - inline bool ShmTransport::IsFullyConnected() const - { - return false; //this->shmCtx.Status() == DepLibSfuShm::SHM_WRT_READY; - } - - void ShmTransport::SendRtpPacket(RTC::RtpPacket* packet, onSendCallback* /* cb */) { MS_TRACE(); - if (!IsConnected())//IsFullyConnected()) + if (!IsConnected()) return; // Increase send transmission. Consumer writes RTP packets to shm, nothing else to do here. @@ -288,7 +276,7 @@ namespace RTC { MS_TRACE(); - if (!IsConnected())//IsFullyConnected()) + if (!IsConnected()) return; // Increase send transmission. @@ -300,7 +288,7 @@ namespace RTC { MS_TRACE(); - if (!IsConnected()) //IsFullyConnected()) + if (!IsConnected()) return; // Increase send transmission. diff --git a/worker/src/RTC/Transport.cpp b/worker/src/RTC/Transport.cpp index bef8378561..2537fd8076 100644 --- a/worker/src/RTC/Transport.cpp +++ b/worker/src/RTC/Transport.cpp @@ -1421,7 +1421,7 @@ namespace RTC break; case RTC::Producer::ReceiveRtpPacketResult::RETRANSMISSION: this->recvRtxTransmission.Update(packet); - MS_DEBUG_TAG(rtp, "L@@K recvRtxTransmission.GetPacketCount()=%d", this->recvRtxTransmission.GetPacketCount()); + MS_DEBUG_TAG(rtp, "L@@K recvRtxTransmission.GetPacketCount()=%zu", this->recvRtxTransmission.GetPacketCount()); break; default:; } From 5ce04ed0e5b1dff9b4ae8f6096e2a5027c9e92fe Mon Sep 17 00:00:00 2001 From: Maria Tverdostup Date: Thu, 20 Aug 2020 19:40:18 -0700 Subject: [PATCH 06/11] test ready --- package.json | 2 +- worker/include/DepLibSfuShm.hpp | 4 ++-- worker/mediasoup-worker.gyp | 14 +------------- worker/src/DepLibSfuShm.cpp | 13 ++++++++----- worker/src/RTC/Router.cpp | 9 ++------- worker/src/RTC/RtpDictionaries/RtpParameters.cpp | 12 ++++-------- worker/src/RTC/ShmConsumer.cpp | 2 +- worker/src/RTC/Transport.cpp | 10 +++------- 8 files changed, 22 insertions(+), 44 deletions(-) diff --git a/package.json b/package.json index b8cae670aa..df59e22380 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@livelyvideo/mediasoup", - "version": "3.5.7-shm73-rtx21", + "version": "3.5.7-shm74-rtx1", "description": "Cutting Edge WebRTC Video Conferencing", "contributors": [ "Iñaki Baz Castillo (https://inakibaz.me)", diff --git a/worker/include/DepLibSfuShm.hpp b/worker/include/DepLibSfuShm.hpp index f4fa8a5199..a4cc1d4ffa 100644 --- a/worker/include/DepLibSfuShm.hpp +++ b/worker/include/DepLibSfuShm.hpp @@ -94,7 +94,7 @@ namespace DepLibSfuShm uint64_t AdjustAudioPktSeq(uint64_t seq); void UpdatePktStat(uint64_t seq, uint64_t ts, Media kind); - bool IsVideoSeqUnset() const; + bool IsLastVideoSeqNotSet() const; uint64_t LastVideoTs() const; uint64_t LastVideoSeq() const; @@ -189,7 +189,7 @@ namespace DepLibSfuShm } } - inline bool ShmCtx::IsVideoSeqUnset() const + inline bool ShmCtx::IsLastVideoSeqNotSet() const { return (this->data[1].last_seq == UINT64_UNSET); } diff --git a/worker/mediasoup-worker.gyp b/worker/mediasoup-worker.gyp index e052b983d6..6fef3eb0b7 100644 --- a/worker/mediasoup-worker.gyp +++ b/worker/mediasoup-worker.gyp @@ -317,10 +317,7 @@ [ { 'target_name': 'mediasoup-worker', - 'defines': [ 'SFU_SHM', 'FFNGXSHM_MAX_NUM_CHANNELS=10', 'SFUSHM_AV_MAX_NUM_CHANNELS=10', 'STREAM_SHM_MAX_CHANNELS=10' ], - #'dependencies': [ - #'deps/ff_shm_api/ff_shm_api.gyp:ff_shm_api' - #], + 'defines': [ 'FFNGXSHM_MAX_NUM_CHANNELS=10', 'SFUSHM_AV_MAX_NUM_CHANNELS=10', 'STREAM_SHM_MAX_CHANNELS=10' ], 'sources': [ # C++ source files. 'src/main.cpp', @@ -332,15 +329,6 @@ 'include/RTC/ShmConsumer.hpp', 'include/RTC/ShmTransport.hpp' ], -# 'include_dirs': -# [ -# '/root/build/ff_shm_api/include' -# ], -# 'library_dirs': [ -# '.', -# '/usr/local/lib/' -# ], -# 'libraries': [ '/usr/local/lib/libffngxshm.a' ] 'library_dirs': [ '/usr/local/lib/' ], diff --git a/worker/src/DepLibSfuShm.cpp b/worker/src/DepLibSfuShm.cpp index c298763104..b4071c71e0 100644 --- a/worker/src/DepLibSfuShm.cpp +++ b/worker/src/DepLibSfuShm.cpp @@ -18,8 +18,11 @@ namespace DepLibSfuShm { { -2, "again (SFUSHM_AV_AGAIN)" } }; - - static constexpr uint64_t MaxVideoPktDelay{ 9000 }; // 100ms in samples (90000 sample rate) TODO: redo to hold constant number of picture frames instead? + // 100ms in samples (90000 sample rate) + // TODO: redo to hold constant number of picture frames instead? + // This will not much better because it is just a different time unit, + // and with exception of old queue items we try to wait to assemble the whole picture frame anyway + static constexpr uint64_t MaxVideoPktDelay{ 9000 }; ShmCtx::~ShmCtx() @@ -154,7 +157,7 @@ namespace DepLibSfuShm { // If queue is empty, seqid is in order, and pkt is not a fragment, there is no need to enqueue if (this->videoPktBuffer.empty() && !isChunkFragment - && (IsVideoSeqUnset() + && (IsLastVideoSeqNotSet() || data->first_rtp_seq - 1 == LastVideoSeq())) { //MS_DEBUG_TAG(rtp, "WRITETHRU [seq=%" PRIu64 " ts=%" PRIu64 "] len=%zu chunkstart=%s chunkend=%s qsize=%zu", data->first_rtp_seq, data->rtp_time, data->len, isChunkStart? "1":"0", isChunkEnd? "1":"0", videoPktBuffer.size()); @@ -162,7 +165,7 @@ namespace DepLibSfuShm { } // Add a too old pkt in queue anyway - if (!IsVideoSeqUnset() + if (!IsLastVideoSeqNotSet() && LastVideoTs() > ts && LastVideoTs() - ts > MaxVideoPktDelay) { @@ -199,7 +202,7 @@ namespace DepLibSfuShm { while (it != this->videoPktBuffer.end()) { // First, write out all chunks with expired timestamps. TODO: mark incomplete fragmented pictures as corrupted (feature TBD in shm writer) - if (!IsVideoSeqUnset() + if (!IsLastVideoSeqNotSet() && LastVideoTs() > it->chunk.rtp_time && LastVideoTs() - it->chunk.rtp_time > MaxVideoPktDelay) { diff --git a/worker/src/RTC/Router.cpp b/worker/src/RTC/Router.cpp index bf892f5f38..0dab4b4a29 100644 --- a/worker/src/RTC/Router.cpp +++ b/worker/src/RTC/Router.cpp @@ -9,9 +9,7 @@ #include "RTC/PipeTransport.hpp" #include "RTC/PlainTransport.hpp" #include "RTC/WebRtcTransport.hpp" -#ifdef SFU_SHM - #include "RTC/ShmTransport.hpp" -#endif +#include "RTC/ShmTransport.hpp" namespace RTC { @@ -251,7 +249,6 @@ namespace RTC case Channel::Request::MethodId::ROUTER_CREATE_SHM_TRANSPORT: { -#ifdef SFU_SHM std::string transportId; // This may throw @@ -269,9 +266,7 @@ namespace RTC shmTransport->FillJson(data); request->Accept(data); -#else - request->Accept(); -#endif + break; } diff --git a/worker/src/RTC/RtpDictionaries/RtpParameters.cpp b/worker/src/RTC/RtpDictionaries/RtpParameters.cpp index 236fcc2013..47114096d8 100644 --- a/worker/src/RTC/RtpDictionaries/RtpParameters.cpp +++ b/worker/src/RTC/RtpDictionaries/RtpParameters.cpp @@ -17,10 +17,8 @@ namespace RTC { "simple", RtpParameters::Type::SIMPLE }, { "simulcast", RtpParameters::Type::SIMULCAST }, { "svc", RtpParameters::Type::SVC }, - { "pipe", RtpParameters::Type::PIPE } -#ifdef SFU_SHM - ,{ "shm", RtpParameters::Type::SHM } -#endif + { "pipe", RtpParameters::Type::PIPE }, + { "shm", RtpParameters::Type::SHM } }; std::map RtpParameters::type2String = { @@ -28,10 +26,8 @@ namespace RTC { RtpParameters::Type::SIMPLE, "simple" }, { RtpParameters::Type::SIMULCAST, "simulcast" }, { RtpParameters::Type::SVC, "svc" }, - { RtpParameters::Type::PIPE, "pipe" } -#ifdef SFU_SHM - ,{ RtpParameters::Type::SHM, "shm" } -#endif + { RtpParameters::Type::PIPE, "pipe" }, + { RtpParameters::Type::SHM, "shm" } }; // clang-format on diff --git a/worker/src/RTC/ShmConsumer.cpp b/worker/src/RTC/ShmConsumer.cpp index f11dcd6345..64e18fab0f 100644 --- a/worker/src/RTC/ShmConsumer.cpp +++ b/worker/src/RTC/ShmConsumer.cpp @@ -463,7 +463,7 @@ namespace RTC { uint8_t nal = *(data) & 0x1F; uint8_t marker = *(pktdata + 1) & 0x80; // Marker bit indicates the last or the only NALU in this packet is the end of the picture data - bool begin_picture = (shmCtx->IsVideoSeqUnset() || (ts > shmCtx->LastVideoTs())); // assume that first video pkt starts the picture frame + bool begin_picture = (shmCtx->IsLastVideoSeqNotSet() || (ts > shmCtx->LastVideoTs())); // assume that first video pkt starts the picture frame // Single NAL unit packet if (nal >= 1 && nal <= 23) { diff --git a/worker/src/RTC/Transport.cpp b/worker/src/RTC/Transport.cpp index 2537fd8076..3c72d778c5 100644 --- a/worker/src/RTC/Transport.cpp +++ b/worker/src/RTC/Transport.cpp @@ -20,10 +20,8 @@ #include "RTC/SimulcastConsumer.hpp" #include "RTC/SvcConsumer.hpp" -#ifdef SFU_SHM - #include "RTC/ShmTransport.hpp" - #include "RTC/ShmConsumer.hpp" -#endif +#include "RTC/ShmTransport.hpp" +#include "RTC/ShmConsumer.hpp" #include // webrtc::RtpPacketSendInfo #include // std::ostream_iterator #include // std::multimap @@ -763,10 +761,8 @@ namespace RTC case RTC::RtpParameters::Type::SHM: { -#ifdef SFU_SHM // This may throw. - consumer = new RTC::ShmConsumer(consumerId, producerId, this, request->data, dynamic_cast(this)->ShmCtx()); -#endif + consumer = new RTC::ShmConsumer(consumerId, producerId, this, request->data, dynamic_cast(this)->ShmCtx()); break; } } From 8b1b9b9890b82c4696234775204e97b20f267f72 Mon Sep 17 00:00:00 2001 From: Maria Tverdostup Date: Thu, 20 Aug 2020 19:51:39 -0700 Subject: [PATCH 07/11] v.3.5.7-shm75-rtx1 to test --- lib/Worker.js | 2 +- lib/index.js | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/Worker.js b/lib/Worker.js index 7b0ba86ff7..14b6b7d652 100644 --- a/lib/Worker.js +++ b/lib/Worker.js @@ -73,7 +73,7 @@ class Worker extends EnhancedEventEmitter_1.EnhancedEventEmitter { // options { env: { - MEDIASOUP_VERSION: '3.5.7-shm73' + MEDIASOUP_VERSION: '3.5.7-shm75-rtx1' }, detached: false, // fd 0 (stdin) : Just ignore it. diff --git a/lib/index.js b/lib/index.js index 41f132a308..18413cd824 100644 --- a/lib/index.js +++ b/lib/index.js @@ -17,7 +17,7 @@ exports.types = types; /** * Expose mediasoup version. */ -exports.version = '3.5.7-shm73'; +exports.version = '3.5.7-shm75-rtx1'; /** * Expose parseScalabilityMode() function. */ diff --git a/package.json b/package.json index df59e22380..b3b35d69e0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@livelyvideo/mediasoup", - "version": "3.5.7-shm74-rtx1", + "version": "3.5.7-shm75-rtx1", "description": "Cutting Edge WebRTC Video Conferencing", "contributors": [ "Iñaki Baz Castillo (https://inakibaz.me)", From ff11e0531fbc54bfa82ccab1abea1f58ce048c43 Mon Sep 17 00:00:00 2001 From: Maria Tverdostup Date: Fri, 21 Aug 2020 18:40:07 -0700 Subject: [PATCH 08/11] Version 3.5.7-shm76-rtx1 can be tested --- lib/Worker.js | 2 +- lib/index.js | 2 +- package.json | 2 +- worker/src/DepLibSfuShm.cpp | 7 ++++--- worker/src/RTC/Producer.cpp | 6 +++--- worker/src/RTC/RtpStream.cpp | 12 ------------ worker/src/RTC/RtpStreamRecv.cpp | 6 +++--- worker/src/RTC/RtxStream.cpp | 12 ------------ worker/src/RTC/Transport.cpp | 4 ++-- 9 files changed, 15 insertions(+), 38 deletions(-) diff --git a/lib/Worker.js b/lib/Worker.js index 14b6b7d652..0cb46ab48e 100644 --- a/lib/Worker.js +++ b/lib/Worker.js @@ -73,7 +73,7 @@ class Worker extends EnhancedEventEmitter_1.EnhancedEventEmitter { // options { env: { - MEDIASOUP_VERSION: '3.5.7-shm75-rtx1' + MEDIASOUP_VERSION: '3.5.7-shm76-rtx1' }, detached: false, // fd 0 (stdin) : Just ignore it. diff --git a/lib/index.js b/lib/index.js index 18413cd824..96f264eb99 100644 --- a/lib/index.js +++ b/lib/index.js @@ -17,7 +17,7 @@ exports.types = types; /** * Expose mediasoup version. */ -exports.version = '3.5.7-shm75-rtx1'; +exports.version = '3.5.7-shm76-rtx1'; /** * Expose parseScalabilityMode() function. */ diff --git a/package.json b/package.json index b3b35d69e0..b8e42a601f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@livelyvideo/mediasoup", - "version": "3.5.7-shm75-rtx1", + "version": "3.5.7-shm76-rtx1", "description": "Cutting Edge WebRTC Video Conferencing", "contributors": [ "Iñaki Baz Castillo (https://inakibaz.me)", diff --git a/worker/src/DepLibSfuShm.cpp b/worker/src/DepLibSfuShm.cpp index b4071c71e0..5b3959c043 100644 --- a/worker/src/DepLibSfuShm.cpp +++ b/worker/src/DepLibSfuShm.cpp @@ -160,7 +160,7 @@ namespace DepLibSfuShm { && (IsLastVideoSeqNotSet() || data->first_rtp_seq - 1 == LastVideoSeq())) { - //MS_DEBUG_TAG(rtp, "WRITETHRU [seq=%" PRIu64 " ts=%" PRIu64 "] len=%zu chunkstart=%s chunkend=%s qsize=%zu", data->first_rtp_seq, data->rtp_time, data->len, isChunkStart? "1":"0", isChunkEnd? "1":"0", videoPktBuffer.size()); + MS_DEBUG_TAG(rtp, "WRITETHRU [seq=%" PRIu64 " ts=%" PRIu64 "]", data->first_rtp_seq, data->rtp_time); return EnqueueResult::SHM_Q_PKT_CANWRITETHRU; } @@ -206,7 +206,7 @@ namespace DepLibSfuShm { && LastVideoTs() > it->chunk.rtp_time && LastVideoTs() - it->chunk.rtp_time > MaxVideoPktDelay) { - MS_DEBUG_TAG(rtp, "OLD [seq=%" PRIu64 " delta=%" PRIu64 "] lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, LastVideoTs() - it->chunk.rtp_time, LastVideoTs(), videoPktBuffer.size()); + MS_DEBUG_TAG(rtp, "OLD [seq=%" PRIu64 " Tsdelta=%" PRIu64 "] lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, LastVideoTs() - it->chunk.rtp_time, LastVideoTs(), videoPktBuffer.size()); prev_seq = it->chunk.first_rtp_seq; this->WriteChunk(&it->chunk, Media::VIDEO); it = this->videoPktBuffer.erase(it); @@ -216,7 +216,7 @@ namespace DepLibSfuShm { // Hole in seqIds: wait some time for missing pkt to be retransmitted if (it->chunk.first_rtp_seq > prev_seq + 1) { - MS_DEBUG_TAG(rtp, "HOLE [seq=%" PRIu64 " ts=%" PRIu64 "] prev=%" PRIu64, it->chunk.first_rtp_seq, it->chunk.rtp_time, prev_seq); + MS_DEBUG_TAG(rtp, "HOLE [seq=%" PRIu64 " ts=%" PRIu64 "] prev=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, it->chunk.rtp_time, prev_seq, LastVideoTs(), videoPktBuffer.size()); return; } @@ -273,6 +273,7 @@ namespace DepLibSfuShm { else // non-fragmented chunk { this->WriteChunk(&it->chunk, Media::VIDEO); + MS_DEBUG_TAG(rtp, "FULL [seq=%" PRIu64 " ts=%" PRIu64 "] lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, it->chunk.rtp_time, LastVideoTs(), videoPktBuffer.size()); prev_seq = it->chunk.first_rtp_seq; it = this->videoPktBuffer.erase(it); continue; diff --git a/worker/src/RTC/Producer.cpp b/worker/src/RTC/Producer.cpp index cd1bed59b4..1271185397 100644 --- a/worker/src/RTC/Producer.cpp +++ b/worker/src/RTC/Producer.cpp @@ -44,7 +44,7 @@ namespace RTC this->rtpParameters = RTC::RtpParameters(*jsonRtpParametersIt); std::string s = jsonRtpParametersIt->dump(); - MS_DEBUG_TAG(rtp,"L@@K Producer ctor has RtpParameters: {%s}L@@K", s.c_str()); + MS_DEBUG_TAG(rtp,"Producer ctor RtpParameters: [%s]", s.c_str()); // Evaluate type. this->type = RTC::RtpParameters::GetType(this->rtpParameters); @@ -616,7 +616,7 @@ namespace RTC result = ReceiveRtpPacketResult::RETRANSMISSION; isRtx = true; - MS_DEBUG_TAG(rtp, "L@@K Retransmitted packet received [ssrc:%" PRIu32 " seq:%" PRIu16 " ts:%" PRIu32 "]", + MS_DEBUG_TAG(rtp, "Retransmitted packet received [ssrc:%" PRIu32 " seq:%" PRIu16 " ts:%" PRIu32 "]", packet->GetSsrc(),packet->GetSequenceNumber(), packet->GetTimestamp()); // Process the packet. @@ -1084,7 +1084,7 @@ namespace RTC for (auto& fb : mediaCodec.rtcpFeedback) { - MS_DEBUG_2TAGS(rtp, rtcp, "L@@K mediaCodec.rtcpFeedback: type=%s parameter=%s", fb.type.c_str(), fb.parameter.c_str()); + MS_DEBUG_2TAGS(rtp, rtcp, "mediaCodec.rtcpFeedback: type=%s parameter=%s", fb.type.c_str(), fb.parameter.c_str()); if (!params.useNack && fb.type == "nack" && fb.parameter == "") { diff --git a/worker/src/RTC/RtpStream.cpp b/worker/src/RTC/RtpStream.cpp index e1b04b8ffc..ac208457fb 100644 --- a/worker/src/RTC/RtpStream.cpp +++ b/worker/src/RTC/RtpStream.cpp @@ -205,12 +205,6 @@ namespace RTC packet->GetSsrc(), packet->GetSequenceNumber()); - MS_DEBUG_TAG( - rtp, - "L@@K too bad sequence number, re-syncing RTP [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", - packet->GetSsrc(), - packet->GetSequenceNumber()); - InitSeq(seq); this->maxPacketTs = packet->GetTimestamp(); @@ -223,12 +217,6 @@ namespace RTC "bad sequence number, ignoring packet [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", packet->GetSsrc(), packet->GetSequenceNumber()); - - MS_DEBUG_TAG( - rtp, - "L@@K bad sequence number, ignoring packet [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", - packet->GetSsrc(), - packet->GetSequenceNumber()); this->badSeq = (seq + 1) & (RtpSeqMod - 1); diff --git a/worker/src/RTC/RtpStreamRecv.cpp b/worker/src/RTC/RtpStreamRecv.cpp index a3aeabf2af..2c7b68adf1 100644 --- a/worker/src/RTC/RtpStreamRecv.cpp +++ b/worker/src/RTC/RtpStreamRecv.cpp @@ -188,13 +188,13 @@ namespace RTC if (this->params.useNack) { this->nackGenerator.reset(new RTC::NackGenerator(this)); - MS_DEBUG_TAG(rtp,"L@@K RtpStreamRecv::params.useNack is true, NACK feature enabled"); + MS_DEBUG_TAG(rtp,"RtpStreamRecv::params.useNack is true, NACK feature enabled"); } else { - MS_DEBUG_TAG(rtp,"L@@K RtpStreamRecv::params.useNack is false, NACK feature disabled"); + MS_DEBUG_TAG(rtp,"RtpStreamRecv::params.useNack is false, NACK feature disabled"); } - MS_DEBUG_TAG(rtp, "L@@K RtpStreamRecv::ctor() HasRtx=%s", HasRtx() ? "TRUE" : "FALSE"); + MS_DEBUG_TAG(rtp, "RtpStreamRecv::HasRtx=%s", HasRtx() ? "TRUE" : "FALSE"); // Run the RTP inactivity periodic timer (unless DTX is enabled). if (!this->params.useDtx) diff --git a/worker/src/RTC/RtxStream.cpp b/worker/src/RTC/RtxStream.cpp index 2e1ea20c82..fe6b0b2efb 100644 --- a/worker/src/RTC/RtxStream.cpp +++ b/worker/src/RTC/RtxStream.cpp @@ -191,12 +191,6 @@ namespace RTC packet->GetSsrc(), packet->GetSequenceNumber()); - MS_DEBUG_TAG( - rtx, - "L@@K RTX too bad sequence number, re-syncing RTP [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", - packet->GetSsrc(), - packet->GetSequenceNumber()); - InitSeq(seq); this->maxPacketTs = packet->GetTimestamp(); @@ -210,12 +204,6 @@ namespace RTC packet->GetSsrc(), packet->GetSequenceNumber()); - MS_DEBUG_TAG( - rtx, - "L@@K RTX bad sequence number, ignoring packet [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", - packet->GetSsrc(), - packet->GetSequenceNumber()); - this->badSeq = (seq + 1) & (RtpSeqMod - 1); // Packet discarded due to late or early arriving. diff --git a/worker/src/RTC/Transport.cpp b/worker/src/RTC/Transport.cpp index 3c72d778c5..e430fccbaa 100644 --- a/worker/src/RTC/Transport.cpp +++ b/worker/src/RTC/Transport.cpp @@ -1417,7 +1417,7 @@ namespace RTC break; case RTC::Producer::ReceiveRtpPacketResult::RETRANSMISSION: this->recvRtxTransmission.Update(packet); - MS_DEBUG_TAG(rtp, "L@@K recvRtxTransmission.GetPacketCount()=%zu", this->recvRtxTransmission.GetPacketCount()); + MS_DEBUG_TAG(rtp, "recvRtxTransmission.GetPacketCount()=%zu", this->recvRtxTransmission.GetPacketCount()); break; default:; } @@ -2383,7 +2383,7 @@ namespace RTC this->sendRtxTransmission.Update(packet); - MS_DEBUG_TAG(rtp, "L@@K sendRtxTransmission.GetPacketCount()=%d", this->sendRtxTransmission.GetPacketCount()); + MS_DEBUG_TAG(rtp, "sendRtxTransmission.GetPacketCount()=%d", this->sendRtxTransmission.GetPacketCount()); } inline void Transport::OnConsumerKeyFrameRequested(RTC::Consumer* consumer, uint32_t mappedSsrc) From 1496e4f4bc88a58c5c9f4bac7a503faae71e63b7 Mon Sep 17 00:00:00 2001 From: Maria Tverdostup Date: Fri, 18 Sep 2020 14:58:27 -0700 Subject: [PATCH 09/11] Ready for testing v.3.5.7-rtx1 --- package.json | 2 +- worker/include/DepLibSfuShm.hpp | 8 ++-- worker/include/Settings.hpp | 1 + worker/src/DepLibSfuShm.cpp | 70 ++++++++++++++++++--------------- worker/src/RTC/ShmConsumer.cpp | 32 +++++++-------- 5 files changed, 58 insertions(+), 55 deletions(-) diff --git a/package.json b/package.json index b8e42a601f..5c85f4d81e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@livelyvideo/mediasoup", - "version": "3.5.7-shm76-rtx1", + "version": "3.5.7-rtx1", "description": "Cutting Edge WebRTC Video Conferencing", "contributors": [ "Iñaki Baz Castillo (https://inakibaz.me)", diff --git a/worker/include/DepLibSfuShm.hpp b/worker/include/DepLibSfuShm.hpp index a4cc1d4ffa..6a11a390bb 100644 --- a/worker/include/DepLibSfuShm.hpp +++ b/worker/include/DepLibSfuShm.hpp @@ -38,10 +38,10 @@ namespace DepLibSfuShm struct MediaState { uint64_t last_seq{ UINT64_UNSET }; // last RTP pkt sequence processed by this input - uint64_t last_ts{ UINT64_UNSET }; // the timestamp of the last processed RTP messpktage - uint32_t ssrc{ 0 }; // ssrc of audio chn - bool sr_received{ false }; - bool sr_written{ false }; + uint64_t last_ts{ UINT64_UNSET }; // the timestamp of the last processed RTP message + uint32_t ssrc{ 0 }; // ssrc of audio or video chn + bool sr_received{ false }; // if sender report was received + bool sr_written{ false }; // if sender report was written into shm uint32_t sr_ntp_msb{ 0 }; uint32_t sr_ntp_lsb{ 0 }; uint64_t sr_rtp_tm{ 0 }; diff --git a/worker/include/Settings.hpp b/worker/include/Settings.hpp index 7b95c7c5ce..7036bd2b2e 100644 --- a/worker/include/Settings.hpp +++ b/worker/include/Settings.hpp @@ -25,6 +25,7 @@ class Settings bool simulcast{ false }; bool svc{ false }; bool sctp{ false }; + bool xcode{ false }; }; public: diff --git a/worker/src/DepLibSfuShm.cpp b/worker/src/DepLibSfuShm.cpp index 5b3959c043..94a1b32416 100644 --- a/worker/src/DepLibSfuShm.cpp +++ b/worker/src/DepLibSfuShm.cpp @@ -28,9 +28,12 @@ namespace DepLibSfuShm { ShmCtx::~ShmCtx() { // Call if writer is not closed - if (SHM_WRT_CLOSED != wrt_status && SHM_WRT_UNDEFINED != wrt_status) + if (SHM_WRT_READY == wrt_status) { - sfushm_av_close_writer(wrt_ctx, 0); + if ( wrt_ctx == nullptr) + MS_WARN_TAG(xcode, "Warning: shm writer context is null but shm is still in ready state."); + else + sfushm_av_close_writer(wrt_ctx, 0); } } @@ -38,9 +41,12 @@ namespace DepLibSfuShm { void ShmCtx::CloseShmWriterCtx() { // Call if writer is not closed - if (SHM_WRT_CLOSED != wrt_status) + if (SHM_WRT_READY == wrt_status) { - sfushm_av_close_writer(wrt_ctx, 0); + if ( wrt_ctx == nullptr) + MS_WARN_TAG(xcode, "Warning: shm writer context is null but shm is still in ready state."); + else + sfushm_av_close_writer(wrt_ctx, 0); } wrt_status = SHM_WRT_CLOSED; @@ -73,7 +79,7 @@ namespace DepLibSfuShm { // Just switched into ready status, can open a writer int err = SFUSHM_AV_OK; if ((err = sfushm_av_open_writer( &wrt_init, &wrt_ctx)) != SFUSHM_AV_OK) { - MS_DEBUG_TAG(rtp, "FAILED in sfushm_av_open_writer() to initialize sfu shm %s with error %s", this->wrt_init.stream_name, GetErrorString(err)); + MS_DEBUG_TAG(xcode, "FAILED in sfushm_av_open_writer() to initialize sfu shm %s with error %s", this->wrt_init.stream_name, GetErrorString(err)); this->wrt_status = SHM_WRT_UNDEFINED; } else @@ -93,7 +99,7 @@ namespace DepLibSfuShm { if (this->CanWrite()) { - MS_DEBUG_TAG(rtp, "shm is ready and first SRs received and written"); + MS_DEBUG_TAG(xcode, "shm is ready and first SRs received and written"); this->listener->OnShmWriterReady(); } } @@ -160,7 +166,7 @@ namespace DepLibSfuShm { && (IsLastVideoSeqNotSet() || data->first_rtp_seq - 1 == LastVideoSeq())) { - MS_DEBUG_TAG(rtp, "WRITETHRU [seq=%" PRIu64 " ts=%" PRIu64 "]", data->first_rtp_seq, data->rtp_time); + MS_DEBUG_TAG(xcode, "WRITETHRU [seq=%" PRIu64 " ts=%" PRIu64 "]", data->first_rtp_seq, data->rtp_time); return EnqueueResult::SHM_Q_PKT_CANWRITETHRU; } @@ -169,7 +175,7 @@ namespace DepLibSfuShm { && LastVideoTs() > ts && LastVideoTs() - ts > MaxVideoPktDelay) { - MS_DEBUG_TAG(rtp, "ENQUEUE OLD seqid=%" PRIu64 " delta=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", data->first_rtp_seq, LastVideoTs() - data->rtp_time, LastVideoTs(), videoPktBuffer.size()); + MS_DEBUG_TAG(xcode, "ENQUEUE OLD seqid=%" PRIu64 " delta=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", data->first_rtp_seq, LastVideoTs() - data->rtp_time, LastVideoTs(), videoPktBuffer.size()); } // Enqueue pkt, newest at the end @@ -206,7 +212,7 @@ namespace DepLibSfuShm { && LastVideoTs() > it->chunk.rtp_time && LastVideoTs() - it->chunk.rtp_time > MaxVideoPktDelay) { - MS_DEBUG_TAG(rtp, "OLD [seq=%" PRIu64 " Tsdelta=%" PRIu64 "] lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, LastVideoTs() - it->chunk.rtp_time, LastVideoTs(), videoPktBuffer.size()); + MS_DEBUG_TAG(xcode, "OLD [seq=%" PRIu64 " Tsdelta=%" PRIu64 "] lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, LastVideoTs() - it->chunk.rtp_time, LastVideoTs(), videoPktBuffer.size()); prev_seq = it->chunk.first_rtp_seq; this->WriteChunk(&it->chunk, Media::VIDEO); it = this->videoPktBuffer.erase(it); @@ -216,7 +222,7 @@ namespace DepLibSfuShm { // Hole in seqIds: wait some time for missing pkt to be retransmitted if (it->chunk.first_rtp_seq > prev_seq + 1) { - MS_DEBUG_TAG(rtp, "HOLE [seq=%" PRIu64 " ts=%" PRIu64 "] prev=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, it->chunk.rtp_time, prev_seq, LastVideoTs(), videoPktBuffer.size()); + MS_DEBUG_TAG(xcode, "HOLE [seq=%" PRIu64 " ts=%" PRIu64 "] prev=%" PRIu64 " lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, it->chunk.rtp_time, prev_seq, LastVideoTs(), videoPktBuffer.size()); return; } @@ -258,7 +264,7 @@ namespace DepLibSfuShm { this->WriteChunk(&chunkStartIt->chunk, Media::VIDEO); chunkStartIt = this->videoPktBuffer.erase(chunkStartIt); } - MS_DEBUG_TAG(rtp, "FRAG start [seq=%" PRIu64 " ts=%" PRIu64 "] end [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", startSeqId, startTs, endSeqId, endTs, videoPktBuffer.size()); + MS_DEBUG_TAG(xcode, "FRAG start [seq=%" PRIu64 " ts=%" PRIu64 "] end [seq=%" PRIu64 " ts=%" PRIu64 "] qsize=%zu", startSeqId, startTs, endSeqId, endTs, videoPktBuffer.size()); chunkStartFound = false; prev_seq = endSeqId; it = nextit; // restore iterator @@ -273,7 +279,7 @@ namespace DepLibSfuShm { else // non-fragmented chunk { this->WriteChunk(&it->chunk, Media::VIDEO); - MS_DEBUG_TAG(rtp, "FULL [seq=%" PRIu64 " ts=%" PRIu64 "] lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, it->chunk.rtp_time, LastVideoTs(), videoPktBuffer.size()); + MS_DEBUG_TAG(xcode, "FULL [seq=%" PRIu64 " ts=%" PRIu64 "] lastTs=%" PRIu64 " qsize=%zu", it->chunk.first_rtp_seq, it->chunk.rtp_time, LastVideoTs(), videoPktBuffer.size()); prev_seq = it->chunk.first_rtp_seq; it = this->videoPktBuffer.erase(it); continue; @@ -304,7 +310,8 @@ namespace DepLibSfuShm { { if (!this->CanWrite()) { - MS_WARN_TAG(rtp, "Cannot write chunk ssrc=%" PRIu32 " seq=%" PRIu64 " ts=%" PRIu64 " because shm writer is not initialized"); + MS_WARN_TAG(xcode, "Cannot write chunk ssrc=%" PRIu32 " seq=%" PRIu64 " ts=%" PRIu64 " because shm writer is not initialized", + data->ssrc, data->first_rtp_seq, data->rtp_time); return; } @@ -319,7 +326,7 @@ namespace DepLibSfuShm { } if (IsError(err)) - MS_WARN_TAG(rtp, "ERROR writing chunk ssrc=%" PRIu32 " seq=%" PRIu64 " ts=%" PRIu64 " to shm: %d - %s", data->ssrc, data->first_rtp_seq, data->rtp_time, err, GetErrorString(err)); + MS_WARN_TAG(xcode, "ERROR writing chunk ssrc=%" PRIu32 " seq=%" PRIu64 " ts=%" PRIu64 " to shm: %d - %s", data->ssrc, data->first_rtp_seq, data->rtp_time, err, GetErrorString(err)); } @@ -332,8 +339,7 @@ namespace DepLibSfuShm { nanoseconds. It is relative to an arbitrary time in the past. It is not related to the time of day and therefore not subject to clock drift. The primary use is for measuring performance between intervals. */ - /* - Uncomment for debugging + uint64_t walltime = DepLibUV::GetTimeMs(); struct timespec clockTime; time_t ct; @@ -343,14 +349,6 @@ namespace DepLibSfuShm { ct = clockTime.tv_sec; uint64_t clockTimeMs = (clockTime.tv_sec * (uint64_t) 1e9 + clockTime.tv_nsec) / 1000000.0; - MS_DEBUG_TAG(rtp, "RTCP SR: SSRC=%d ReportNTP(ms)=%" PRIu64 " RtpTs=%" PRIu32 " uv_hrtime(ms)=%" PRIu64 " clock_gettime(s)=%" PRIu64 " clock_gettime(ms)=%" PRIu64, - ssrc, - lastSenderReportNtpMs, - lastSenderReporTs, - walltime, - ct, - clockTimeMs); - */ auto ntp = Utils::Time::TimeMs2Ntp(lastSenderReportNtpMs); size_t idx = (kind == Media::AUDIO) ? 0 : 1; @@ -359,6 +357,14 @@ namespace DepLibSfuShm { this->data[idx].sr_ntp_lsb = ntp.fractions; this->data[idx].sr_rtp_tm = lastSenderReporTs; + MS_DEBUG_TAG(xcode, "Received RTCP SR: SSRC=%" PRIu32 " ReportNTP(ms)=%" PRIu64 " RtpTs=%" PRIu32 " uv_hrtime(ms)=%" PRIu64 " clock_gettime(s)=%" PRIu64 " clock_gettime(ms)=%" PRIu64, + this->data[idx].ssrc, + lastSenderReportNtpMs, + lastSenderReporTs, + walltime, + ct, + clockTimeMs); + if (SHM_WRT_READY != this->wrt_status) return; // shm writer is not set up yet @@ -383,7 +389,7 @@ namespace DepLibSfuShm { if (shouldNotify) { - MS_DEBUG_TAG(rtp, "First SRs received, and shm is ready"); + MS_DEBUG_TAG(xcode, "First SRs received, and shm is ready"); this->listener->OnShmWriterReady(); } } @@ -395,7 +401,7 @@ namespace DepLibSfuShm { int err = sfushm_av_write_rtcp_sr_ts(wrt_ctx, this->data[idx].sr_ntp_msb, this->data[idx].sr_ntp_lsb, this->data[idx].sr_rtp_tm, this->data[idx].ssrc); if (IsError(err)) { - MS_WARN_TAG(rtp, "ERROR writing RTCP SR for ssrc %" PRIu32 " %d - %s", this->data[idx].ssrc, err, GetErrorString(err)); + MS_WARN_TAG(xcode, "ERROR writing RTCP SR for ssrc %" PRIu32 " %d - %s", this->data[idx].ssrc, err, GetErrorString(err)); } } @@ -404,7 +410,7 @@ namespace DepLibSfuShm { { if (SHM_WRT_READY != this->wrt_status) { - MS_WARN_TAG(rtp, "Cannot write stream metadata because shm writer is not initialized"); + MS_WARN_TAG(xcode, "Cannot write stream metadata because shm writer is not initialized"); return -1; // shm writer is not set up yet } @@ -413,20 +419,20 @@ namespace DepLibSfuShm { if (0 != this->stream_name.compare(shm)) { - MS_WARN_TAG(rtp, "input metadata shm name '%s' does not match '%s'", shm.c_str(), this->stream_name.c_str()); + MS_WARN_TAG(xcode, "input metadata shm name '%s' does not match '%s'", shm.c_str(), this->stream_name.c_str()); return -1; } if (metadata.length() > 255) { - MS_WARN_TAG(rtp, "input metadata is too long: %s", metadata.c_str()); + MS_WARN_TAG(xcode, "input metadata is too long: %s", metadata.c_str()); } std::copy(metadata.begin(), metadata.end(), data); err = sfushm_av_write_stream_metadata(wrt_ctx, data, metadata.length()); if (IsError(err)) { - MS_WARN_TAG(rtp, "ERROR writing stream metadata: %d - %s", err, GetErrorString(err)); + MS_WARN_TAG(xcode, "ERROR writing stream metadata: %d - %s", err, GetErrorString(err)); return -1; } @@ -438,13 +444,13 @@ namespace DepLibSfuShm { { if (SHM_WRT_READY != this->wrt_status) { - MS_WARN_TAG(rtp, "Cannot write video rotation because shm writer is not initialized"); + MS_WARN_TAG(xcode, "Cannot write video rotation because shm writer is not initialized"); return; // shm writer is not set up yet } int err = sfushm_av_write_video_rotation(wrt_ctx, rotation); if (IsError(err)) - MS_WARN_TAG(rtp, "ERROR writing video rotation: %d - %s", err, GetErrorString(err)); + MS_WARN_TAG(xcode, "ERROR writing video rotation: %d - %s", err, GetErrorString(err)); } } // namespace DepLibSfuShm \ No newline at end of file diff --git a/worker/src/RTC/ShmConsumer.cpp b/worker/src/RTC/ShmConsumer.cpp index 64e18fab0f..7618b0a380 100644 --- a/worker/src/RTC/ShmConsumer.cpp +++ b/worker/src/RTC/ShmConsumer.cpp @@ -199,7 +199,7 @@ namespace RTC if (!this->producerRtpStream || !this->producerRtpStream->GetSenderReportNtpMs() || !this->producerRtpStream->GetSenderReportTs()) { - MS_DEBUG_TAG(rtp, "Producer stream failed to read SR RTCP msg"); + MS_DEBUG_2TAGS(rtcp, xcode, "Producer stream failed to read SR RTCP msg"); return; } @@ -285,7 +285,7 @@ namespace RTC // Check for video orientation changes if (VideoOrientationChanged(packet)) { - MS_DEBUG_TAG(rtp, "Video orientation changed to %d in packet[ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + MS_DEBUG_2TAGS(rtp, xcode, "Video orientation changed to %d in packet[ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", this->rotation, packet->GetSsrc(), packet->GetSequenceNumber(), @@ -468,7 +468,7 @@ namespace RTC if (nal >= 1 && nal <= 23) { if (len < 1) { - MS_WARN_TAG(rtp, "NALU data len < 1: %lu", len); + MS_WARN_TAG(xcode, "NALU data len < 1: %lu", len); break; } @@ -499,10 +499,9 @@ namespace RTC this->chunk.begin = begin_picture; this->chunk.end = (marker != 0); - //if (nal != 1) if (isKeyFrame) { - MS_DEBUG_TAG(rtp, "video single NALU=%d LEN=%zu ts %" PRIu64 " seq %" PRIu64 " isKeyFrame=%d begin_picture(chunk.begin)=%d marker(chunk.end)=%d lastTs=%" PRIu64, + MS_DEBUG_TAG(xcode, "video single NALU=%d LEN=%zu ts %" PRIu64 " seq %" PRIu64 " isKeyFrame=%d begin_picture(chunk.begin)=%d marker(chunk.end)=%d lastTs=%" PRIu64, nal, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, isKeyFrame, begin_picture, marker, shmCtx->LastVideoTs()); } @@ -541,7 +540,7 @@ namespace RTC { uint16_t naluSize = Utils::Byte::Get2Bytes(data, offset); if ( offset + naluSize > len) { - MS_WARN_TAG(rtp, "payload left to read from STAP-A is too short: %zu > %zu", offset + naluSize, len); + MS_WARN_TAG(xcode, "payload left to read from STAP-A is too short: %zu > %zu", offset + naluSize, len); break; } @@ -576,12 +575,9 @@ namespace RTC this->chunk.first_rtp_seq = this->chunk.last_rtp_seq = seq; this->chunk.ssrc = ssrc; - if (isKeyFrame) - { - MS_DEBUG_TAG(rtp, "video STAP-A: NAL=%d payloadlen=%" PRIu32 " nalulen=%" PRIu16 " chunklen=%" PRIu32 " ts=%" PRIu64 " seq=%" PRIu64 " lastTs=%" PRIu64 " isKeyFrame=%d chunk.begin=%d chunk.end=%d", - subnal, len, naluSize, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, - shmCtx->LastVideoTs(), isKeyFrame, this->chunk.begin, this->chunk.end); - } + MS_DEBUG_TAG(xcode, "video STAP-A: NAL=%d payloadlen=%" PRIu32 " nalulen=%" PRIu16 " chunklen=%" PRIu32 " ts=%" PRIu64 " seq=%" PRIu64 " lastTs=%" PRIu64 " isKeyFrame=%d chunk.begin=%d chunk.end=%d", + subnal, len, naluSize, this->chunk.len, this->chunk.rtp_time, this->chunk.first_rtp_seq, + shmCtx->LastVideoTs(), isKeyFrame, this->chunk.begin, this->chunk.end); shmCtx->WriteRtpDataToShm(DepLibSfuShm::Media::VIDEO, &chunk, !(this->chunk.begin && this->chunk.end)); offset += chunksize; @@ -607,7 +603,7 @@ namespace RTC { if (len < 3) { - MS_DEBUG_TAG(rtp, "FU-A payload too short"); + MS_DEBUG_TAG(xcode, "FU-A payload too short"); break; } // Parse FU header octet @@ -655,9 +651,9 @@ namespace RTC this->chunk.first_rtp_seq = this->chunk.last_rtp_seq = seq; this->chunk.ssrc = ssrc; this->chunk.end = (endBit && marker) ? 1 : 0; - if (isKeyFrame) //(subnal != 1) + if (isKeyFrame) // remove if not afraid of too much output { - MS_DEBUG_TAG(rtp, "video FU-A NAL=%" PRIu8 " len=%" PRIu32 " ts=%" PRIu64 " prev_ts=%" PRIu64 " seq=%" PRIu64 " isKeyFrame=%d startBit=%" PRIu8 " endBit=%" PRIu8 " marker=%" PRIu8 " chunk.begin=%d chunk.end=%d", + MS_DEBUG_TAG(xcode, "video FU-A NAL=%" PRIu8 " len=%" PRIu32 " ts=%" PRIu64 " prev_ts=%" PRIu64 " seq=%" PRIu64 " isKeyFrame=%d startBit=%" PRIu8 " endBit=%" PRIu8 " marker=%" PRIu8 " chunk.begin=%d chunk.end=%d", subnal, this->chunk.len, this->chunk.rtp_time, shmCtx->LastVideoTs(), this->chunk.first_rtp_seq, isKeyFrame, startBit, endBit, marker, this->chunk.begin, this->chunk.end); } @@ -670,12 +666,12 @@ namespace RTC case 27: // MTAP-24 case 29: // FU-B { - MS_WARN_TAG(rtp, "Unsupported NAL unit type %u in video packet", nal); + MS_WARN_TAG(xcode, "Unsupported NAL unit type %u in video packet", nal); break; } default: // ignore the rest { - MS_DEBUG_TAG(rtp, "Unknown NAL unit type %u in video packet", nal); + MS_DEBUG_TAG(xcode, "Unknown NAL unit type %u in video packet", nal); break; } } @@ -924,7 +920,7 @@ namespace RTC { MS_TRACE(); - MS_DEBUG_TAG(rtp, "writer ready, consumer %s, get key frame", IsActive() ? "ready" : "not ready"); + MS_DEBUG_TAG(xcode, "writer ready, consumer %s, get key frame", IsActive() ? "ready" : "not ready"); this->syncRequired = true; From fa0e51b8a81a2f26a9e7b6aaddbd29fb5c2d028e Mon Sep 17 00:00:00 2001 From: Maria Tverdostup Date: Fri, 18 Sep 2020 16:28:33 -0700 Subject: [PATCH 10/11] remove L@@K --- worker/src/RTC/RtpStreamRecv.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/src/RTC/RtpStreamRecv.cpp b/worker/src/RTC/RtpStreamRecv.cpp index 6851623c7b..4730b24a38 100644 --- a/worker/src/RTC/RtpStreamRecv.cpp +++ b/worker/src/RTC/RtpStreamRecv.cpp @@ -331,7 +331,7 @@ namespace RTC MS_DEBUG_TAG( rtx, - "L@@KL@@KL@@K Received RTX packet [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", + "Received RTX packet [ssrc:%" PRIu32 ", seq:%" PRIu16 "]", packet->GetSsrc(), packet->GetSequenceNumber() ); From 36b651d5b983cdc5001ddda948aaa4f116cbc28b Mon Sep 17 00:00:00 2001 From: Maria Tverdostup Date: Fri, 18 Sep 2020 16:38:00 -0700 Subject: [PATCH 11/11] remove unnecessary file --- worker/include/RTC/ShmWriterVideoBuffer.hpp | 40 --------------------- 1 file changed, 40 deletions(-) delete mode 100644 worker/include/RTC/ShmWriterVideoBuffer.hpp diff --git a/worker/include/RTC/ShmWriterVideoBuffer.hpp b/worker/include/RTC/ShmWriterVideoBuffer.hpp deleted file mode 100644 index e7a9268db9..0000000000 --- a/worker/include/RTC/ShmWriterVideoBuffer.hpp +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef MS_RTC_SHM_WRITER_VIDEO_BUFFER_HPP -#define MS_RTC_SHM_WRITER_VIDEO_BUFFER_HPP - -#include "RTC/RtpPacket.hpp" -#include - -namespace RTC -{ - class ShmWriterVideoBuffer // public RTC::RtpStreamSend::Listener - { - public: - struct VideoBufferItem - { - // Cloned packet. TODO: maybe null if currently missing - RTC::RtpPacket* packet{ nullptr }; - // Memory to hold the cloned packet (with extra space for RTX encoding). - uint8_t store[RTC::MtuSize + 100]; - //if it is key frame (or part of key frame in case of fragmented pic) - bool keyFrame; - bool picBegin; - bool picEnd; - bool NaluBegin; - bool NaluEnd; - // written out to shm or discarded; either way, this item can now be removed from the buffer - bool done; - //TODO: keep timestamps and seqIds separately, or rely on packet field? - }; - //ShmWriterVideoBuffer(); - - private: - uint64_t oldestTs; - uint64_t newestTs; - uint16_t oldestSeqId; - uint16_t newestSeqId; - uint32_t ssrc; - std::vector buffer; // TODO: maybe map by seqId? - }; -} - -#endif //MS_RTC_SHM_WRITER_VIDEO_BUFFER_HPP \ No newline at end of file