Skip to content

Implement NACK+RTX suppport #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 11 commits into
base: workflows
Choose a base branch
from
Open

Implement NACK+RTX suppport #45

Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
1b0aeea
Implement NACK+RTX suppport
murillo128 Jun 10, 2025
734463e
avformat/whip: remove extra blank lines
JackLau1222 Jun 10, 2025
2ed9d06
avformat/whip: properly handle encrypted SRTCP NACK pakcet
JackLau1222 Jun 12, 2025
5f688a1
avformat/whip: fix typo
JackLau1222 Jun 12, 2025
890afb8
avformat/whip: fix whip->hist_head overflow
JackLau1222 Jun 13, 2025
809f0a5
avformat/whip: add srtp setup for video rtx
JackLau1222 Jun 14, 2025
6f0ed41
avfotmat/whip: add support to handle multi NACK in a bundled packet
JackLau1222 Jun 15, 2025
278babf
avformat/whip: reindent for adding handle multi nack support
JackLau1222 Jun 15, 2025
3eaedf7
avformat/whip: simplify NACK verification logic
JackLau1222 Jun 15, 2025
34b5479
avformat/whip: fix srtcp buffer crosses boundary
JackLau1222 Jun 15, 2025
5348acd
avformat/whip: optimize log content
JackLau1222 Jun 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
197 changes: 194 additions & 3 deletions libavformat/whip.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@
/* Referring to Chrome's definition of RTP payload types. */
#define WHIP_RTP_PAYLOAD_TYPE_H264 106
#define WHIP_RTP_PAYLOAD_TYPE_OPUS 111
#define WHIP_RTP_PAYLOAD_TYPE_RTX 105

/**
* The STUN message header, which is 20 bytes long, comprises the
Expand Down Expand Up @@ -150,6 +151,11 @@
#define WHIP_SDP_SESSION_ID "4489045141692799359"
#define WHIP_SDP_CREATOR_IP "127.0.0.1"

/**
* Retransmission / NACK support
*/
#define HISTORY_SIZE_DEFAULT 4096

/* Calculate the elapsed time from starttime to endtime in milliseconds. */
#define ELAPSED(starttime, endtime) ((int)(endtime - starttime) / 1000)

Expand Down Expand Up @@ -193,6 +199,15 @@ enum WHIPState {
WHIP_STATE_FAILED,
};

typedef struct RtpHistoryItem {
/* original RTP seq */
uint16_t seq;
/* length in bytes */
int size;
/* malloc-ed copy */
uint8_t* pkt;
} RtpHistoryItem;

typedef struct WHIPContext {
AVClass *av_class;

Expand Down Expand Up @@ -279,6 +294,7 @@ typedef struct WHIPContext {
/* The SRTP send context, to encrypt outgoing packets. */
SRTPContext srtp_audio_send;
SRTPContext srtp_video_send;
SRTPContext srtp_video_rtx_send;
SRTPContext srtp_rtcp_send;
/* The SRTP receive context, to decrypt incoming packets. */
SRTPContext srtp_recv;
Expand All @@ -303,6 +319,15 @@ typedef struct WHIPContext {
/* The certificate and private key used for DTLS handshake. */
char* cert_file;
char* key_file;

/* RTX / NACK */
uint8_t rtx_payload_type;
uint32_t video_rtx_ssrc;
uint16_t rtx_seq;
int history_size;
RtpHistoryItem * history; /* ring buffer */
int hist_head;
int enable_nack_rtx;
} WHIPContext;

/**
Expand Down Expand Up @@ -615,6 +640,17 @@ static int generate_sdp_offer(AVFormatContext *s)
whip->audio_payload_type = WHIP_RTP_PAYLOAD_TYPE_OPUS;
whip->video_payload_type = WHIP_RTP_PAYLOAD_TYPE_H264;

/* RTX / NACK init */
whip->rtx_payload_type = WHIP_RTP_PAYLOAD_TYPE_RTX;
whip->video_rtx_ssrc = av_lfg_get(&whip->rnd);
whip->rtx_seq = 0;
whip->hist_head = 0;
whip->history_size = FFMAX(64, whip->history_size);
whip->history = av_calloc(whip->history_size, sizeof(*whip->history));
if (!whip->history)
return AVERROR(ENOMEM);
whip->enable_nack_rtx = 1;

av_bprintf(&bp, ""
"v=0\r\n"
"o=FFmpeg %s 2 IN IP4 %s\r\n"
Expand Down Expand Up @@ -666,7 +702,7 @@ static int generate_sdp_offer(AVFormatContext *s)
}

av_bprintf(&bp, ""
"m=video 9 UDP/TLS/RTP/SAVPF %u\r\n"
"m=video 9 UDP/TLS/RTP/SAVPF %u %u\r\n"
"c=IN IP4 0.0.0.0\r\n"
"a=ice-ufrag:%s\r\n"
"a=ice-pwd:%s\r\n"
Expand All @@ -679,9 +715,16 @@ static int generate_sdp_offer(AVFormatContext *s)
"a=rtcp-rsize\r\n"
"a=rtpmap:%u %s/90000\r\n"
"a=fmtp:%u level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=%02x%02x%02x\r\n"
"a=rtcp-fb:%u nack\r\n"
"a=rtpmap:%u rtx/90000\r\n"
"a=fmtp:%u apt=%u\r\n"
"a=ssrc-group:FID %u %u\r\n"
"a=ssrc:%u cname:FFmpeg\r\n"
"a=ssrc:%u msid:FFmpeg video\r\n"
"a=ssrc:%u cname:FFmpeg\r\n"
"a=ssrc:%u msid:FFmpeg video\r\n",
whip->video_payload_type,
whip->rtx_payload_type,
whip->ice_ufrag_local,
whip->ice_pwd_local,
whip->dtls_fingerprint,
Expand All @@ -691,8 +734,16 @@ static int generate_sdp_offer(AVFormatContext *s)
profile,
profile_iop,
level,
whip->video_payload_type,
whip->rtx_payload_type,
whip->rtx_payload_type,
whip->video_payload_type,
whip->video_ssrc,
whip->video_rtx_ssrc,
whip->video_ssrc,
whip->video_ssrc,
whip->video_ssrc);
whip->video_rtx_ssrc,
whip->video_rtx_ssrc);
}

if (!av_bprint_is_complete(&bp)) {
Expand Down Expand Up @@ -1392,6 +1443,12 @@ static int setup_srtp(AVFormatContext *s)
goto end;
}

ret = ff_srtp_set_crypto(&whip->srtp_video_rtx_send, suite, buf);
if (ret < 0) {
av_log(whip, AV_LOG_ERROR, "WHIP: Failed to set crypto for video rtx send\n");
goto end;
}

ret = ff_srtp_set_crypto(&whip->srtp_rtcp_send, suite, buf);
if (ret < 0) {
av_log(whip, AV_LOG_ERROR, "Failed to set crypto for rtcp send\n");
Expand Down Expand Up @@ -1421,6 +1478,37 @@ static int setup_srtp(AVFormatContext *s)
return ret;
}


/**
* RTX history helpers
*/
static void rtp_history_store(WHIPContext *whip, const uint8_t *pkt, int size)
{
int pos = whip->hist_head % whip->history_size;
RtpHistoryItem * it = &whip->history[pos];
/* free older entry */
av_free(it->pkt);
it->pkt = av_malloc(size);
if (!it->pkt)
return;

memcpy(it->pkt, pkt, size);
it->size = size;
it->seq = AV_RB16(pkt + 2);

whip->hist_head = ++pos;
}

static const RtpHistoryItem* rtp_history_find(const WHIPContext *whip, uint16_t seq)
{
for (int i = 0; i < whip->history_size; i++) {
const RtpHistoryItem * it = &whip->history[i];
if (it->pkt && it->seq == seq)
return it;
}
return NULL;
}

/**
* Callback triggered by the RTP muxer when it creates and sends out an RTP packet.
*
Expand Down Expand Up @@ -1457,6 +1545,10 @@ static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size)
return 0;
}

/* Store only ORIGINAL video packets (non-RTX, non-RTCP) */
if (!is_rtcp && is_video)
rtp_history_store(whip, buf, buf_size);

ret = ffurl_write(whip->udp, whip->buf, cipher_size);
if (ret < 0) {
av_log(whip, AV_LOG_ERROR, "WHIP: Failed to write packet=%dB, ret=%d\n", cipher_size, ret);
Expand All @@ -1465,6 +1557,46 @@ static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size)

return ret;
}
/**
* See https://datatracker.ietf.org/doc/html/rfc4588
* Build and send a single RTX packet
*/
static int send_rtx_packet(AVFormatContext *s, const uint8_t * orig_pkt, int orig_size)
{
WHIPContext * whip = s->priv_data;
int new_size, cipher_size;
/* skip if no RTX PT configured */
if (!whip->enable_nack_rtx)
return 0;

/* allocate new buffer: header + 2 + payload */
if (orig_size + 2 > sizeof(whip->buf))
return 0;

memcpy(whip->buf, orig_pkt, orig_size);

uint8_t * hdr = whip->buf;
uint16_t orig_seq = AV_RB16(hdr + 2);

/* rewrite header */
hdr[1] = (hdr[1] & 0x80) | whip->rtx_payload_type; /* keep M bit */
AV_WB16(hdr + 2, whip->rtx_seq++);
AV_WB32(hdr + 8, whip->video_rtx_ssrc);

/* shift payload 2 bytes */
memmove(hdr + 12 + 2, hdr + 12, orig_size - 12);
AV_WB16(hdr + 12, orig_seq);

new_size = orig_size + 2;

/* Encrypt by SRTP and send out. */
cipher_size = ff_srtp_encrypt(&whip->srtp_video_rtx_send, whip->buf, new_size, whip->buf, sizeof(whip->buf));
if (cipher_size <= 0 || cipher_size < new_size) {
av_log(whip, AV_LOG_WARNING, "WHIP: Failed to encrypt packet=%dB, cipher=%dB\n", new_size, cipher_size);
return 0;
}
return ffurl_write(whip->udp, whip->buf, cipher_size);
}

/**
* Creates dedicated RTP muxers for each stream in the AVFormatContext to build RTP
Expand Down Expand Up @@ -1789,6 +1921,64 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt)
goto end;
}
}
/**
* Handle RTCP NACK
* Refer to RFC 4585, Section 6.2.1
* The Generic NACK message is identified by PT=RTPFB and FMT=1.
* TODO: disable retransmisstion when "-tune zerolatency"
*/
if (media_is_rtcp(whip->buf, ret)) {
int ptr = 0;
uint8_t pt = whip->buf[ptr + 1];
uint8_t fmt = (whip->buf[ptr] & 0x1f);
if (ptr + 4 <= ret && pt == 205 && fmt == 1 ) {
/**
* Refer to RFC 3550, Section 6.4.1.
* The length of this RTCP packet in 32-bit words minus one,
* including the header and any padding.
*/
int len = (AV_RB16(&whip->buf[ptr + 2]) + 1) * 4;
if (ptr + len < ret && len >= 12) {
int i = 0;
/* SRTCP index(4 bytes) + HMAC (SRTP_AES128_CM_SHA1_80 10bytes) */
int srtcp_len = len + 4 + 10;
uint8_t *pkt = av_malloc(srtcp_len);
memcpy(pkt, whip->buf, srtcp_len);
int ret = ff_srtp_decrypt(&whip->srtp_recv, pkt, &srtcp_len);
if (ret < 0)
av_log(whip, AV_LOG_ERROR, "WHIP: SRTCP decrypt failed: %d\n", ret);
while (12 + i < len && ret >= 0) {
/**
* See https://datatracker.ietf.org/doc/html/rfc4585#section-6.1
* Handle multi NACKs in bundled packet.
*/
uint16_t pid = AV_RB16(&pkt[ptr + 12 + i]);
uint16_t blp = AV_RB16(&pkt[ptr + 14 + i]);

/* retransmit pid + any bit set in blp */
for (int bit = -1; bit < 16; bit++) {
uint16_t seq = (bit < 0) ? pid : pid + bit + 1;
if (bit >= 0 && !(blp & (1 << bit)))
continue;

const RtpHistoryItem * it = rtp_history_find(whip, seq);
if (it) {
ret = send_rtx_packet(s, it->pkt, it->size);
av_log(whip, AV_LOG_INFO,
"WHIP: NACK, packet found: size: %d, seq=%d, rtx size=%d, lateset stored packet seq:%d\n",
it->size, seq, ret, whip->history[whip->hist_head-1].seq);
} else {
av_log(whip, AV_LOG_INFO,
"WHIP: NACK, packet not found, seq=%d, blp=%d, latest stored packet seq: %d, latest rtx seq: %d\n",
seq, blp, whip->history[whip->hist_head-1].seq, whip->rtx_seq);
}
}
i = i + 4;
}
av_free(pkt);
}
}
}
} else if (ret != AVERROR(EAGAIN)) {
av_log(whip, AV_LOG_ERROR, "WHIP: Failed to read from UDP socket\n");
goto end;
Expand Down Expand Up @@ -1891,7 +2081,8 @@ static const AVOption options[] = {
{ "pkt_size", "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size), AV_OPT_TYPE_INT, { .i64 = 1200 }, -1, INT_MAX, DEC },
{ "authorization", "The optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC },
{ "cert_file", "The optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC },
{ "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC },
{ "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC },
{ "rtx_history", "Packet history size", OFFSET(history_size), AV_OPT_TYPE_INT, { .i64 = HISTORY_SIZE_DEFAULT }, 64, INT_MAX, DEC },
{ NULL },
};

Expand Down