Skip to content

Feature: add ICE lite support and NACK support for WHIP #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Feature: add ICE lite support and NACK support for WHIP #23

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3679593
Implement ICE lite support and consent freshness
murillo128 Jun 5, 2025
9059b58
Implement nack+rtx support for video
murillo128 Jun 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
231 changes: 226 additions & 5 deletions libavformat/whip.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
#include "network.h"
#include "srtp.h"
#include "tls.h"

#include <errno.h>
/**
* Maximum size limit of a Session Description Protocol (SDP),
* be it an offer or answer.
Expand Down Expand Up @@ -114,6 +114,7 @@
/* Referring to Chrome's definition of RTP payload types. */
#define WHIP_RTP_PAYLOAD_TYPE_H264 106
#define WHIP_RTP_PAYLOAD_TYPE_OPUS 111
#define WHIP_RTP_PAYLOAD_TYPE_RTX 105

/**
* The STUN message header, which is 20 bytes long, comprises the
Expand Down Expand Up @@ -143,22 +144,35 @@
#define WHIP_RTCP_PT_START 192
#define WHIP_RTCP_PT_END 223

/**
* Consent-freshness constants �����������������������������������������
*/
#define WHIP_CONSENT_DEF_INTERVAL 15000 /* ms � RFC 7675 default */
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A special char �

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@murillo128 we need your help

#define WHIP_CONSENT_MAX_FAILURES 3

/**
* In the case of ICE-LITE, these fields are not used; instead, they are defined
* as constant values.
*/
#define WHIP_SDP_SESSION_ID "4489045141692799359"
#define WHIP_SDP_CREATOR_IP "127.0.0.1"

/**
* Retransmission / NACK support
*/
#define HISTORY_SIZE_DEFAULT 512

/* Calculate the elapsed time from starttime to endtime in milliseconds. */
#define ELAPSED(starttime, endtime) ((int)(endtime - starttime) / 1000)

/* STUN Attribute, comprehension-required range (0x0000-0x7FFF) */
enum STUNAttr {
STUN_ATTR_USERNAME = 0x0006, /// shared secret response/bind request
STUN_ATTR_PRIORITY = 0x0024, /// ICE controlling/controlled
STUN_ATTR_USE_CANDIDATE = 0x0025, /// bind request
STUN_ATTR_MESSAGE_INTEGRITY = 0x0008, /// bind request/response
STUN_ATTR_FINGERPRINT = 0x8028, /// rfc5389
STUN_ATTR_ICE_CONTROLLING = 0x802A, /// full agent talking to ice-lite
};

enum WHIPState {
Expand Down Expand Up @@ -193,6 +207,15 @@ enum WHIPState {
WHIP_STATE_FAILED,
};

typedef struct RtpHistoryItem {
/* original RTP seq */
uint16_t seq;
/* length in bytes */
int size;
/* malloc-ed copy */
uint8_t* pkt;
} RtpHistoryItem;

typedef struct WHIPContext {
AVClass *av_class;

Expand Down Expand Up @@ -303,6 +326,25 @@ typedef struct WHIPContext {
/* The certificate and private key used for DTLS handshake. */
char* cert_file;
char* key_file;

/* Consent-freshness state */
int consent_interval;
int64_t last_consent_tx;
int consent_failures;

/* ICE-lite support */
int ice_lite_remote;
uint64_t ice_tie_breaker; /* random 64-bit, for ICE-CONTROLLING */

/* RTX / NACK */
uint8_t rtx_payload_type;
uint32_t video_rtx_ssrc;
uint16_t rtx_seq;
int history_size;
RtpHistoryItem * history; /* ring buffer */
int hist_head;
int enable_nack_rtx;

} WHIPContext;

/**
Expand Down Expand Up @@ -412,6 +454,15 @@ static av_cold int initialize(AVFormatContext *s)
seed = av_get_random_seed();
av_lfg_init(&whip->rnd, seed);

/* 64-bit tie-breaker for ICE-CONTROLLING (RFC 8445 6.1.1) */
whip->ice_tie_breaker = ((uint64_t)av_lfg_get(&whip->rnd) << 32) | (uint64_t)av_lfg_get(&whip->rnd);

/* Initialise consent-freshness timers */
if (whip->consent_interval <= 0)
whip->consent_interval = WHIP_CONSENT_DEF_INTERVAL;
whip->last_consent_tx = av_gettime();
whip->consent_failures = 0;

if (whip->pkt_size < ideal_pkt_size)
av_log(whip, AV_LOG_WARNING, "WHIP: pkt_size=%d(<%d) is too small, may cause packet loss\n",
whip->pkt_size, ideal_pkt_size);
Expand Down Expand Up @@ -615,6 +666,17 @@ static int generate_sdp_offer(AVFormatContext *s)
whip->audio_payload_type = WHIP_RTP_PAYLOAD_TYPE_OPUS;
whip->video_payload_type = WHIP_RTP_PAYLOAD_TYPE_H264;

/* RTX / NACK init */
whip->rtx_payload_type = WHIP_RTP_PAYLOAD_TYPE_RTX;
whip->video_rtx_ssrc = av_lfg_get(&whip->rnd);
whip->rtx_seq = 0;
whip->hist_head = 0;
whip->history_size = FFMAX(64, whip->history_size);
whip->history = av_calloc(whip->history_size, sizeof(*whip->history));
if (!whip->history)
return AVERROR(ENOMEM);
whip->enable_nack_rtx = 1;

av_bprintf(&bp, ""
"v=0\r\n"
"o=FFmpeg %s 2 IN IP4 %s\r\n"
Expand Down Expand Up @@ -679,8 +741,14 @@ static int generate_sdp_offer(AVFormatContext *s)
"a=rtcp-rsize\r\n"
"a=rtpmap:%u %s/90000\r\n"
"a=fmtp:%u level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=%02x%02x%02x\r\n"
"a=rtcp-fb:%u nack\r\n"
"a=rtpmap:%u rtx/90000\r\n"
"a=fmtp:%u apt=%u\r\n"
"a=ssrc:%u cname:FFmpeg\r\n"
"a=ssrc:%u msid:FFmpeg video\r\n"
"a=ssrc:%u cname:FFmpeg\r\n"
"a=ssrc:%u msid:FFmpeg video\r\n",
"a=ssrc:%u msid:FFmpeg video\r\n"
"a=ssrc-group:FID %u %u\r\n",
whip->video_payload_type,
whip->ice_ufrag_local,
whip->ice_pwd_local,
Expand All @@ -691,8 +759,16 @@ static int generate_sdp_offer(AVFormatContext *s)
profile,
profile_iop,
level,
whip->video_payload_type,
whip->rtx_payload_type,
whip->rtx_payload_type,
whip->video_payload_type,
whip->video_ssrc,
whip->video_ssrc,
whip->video_ssrc);
whip->video_rtx_ssrc,
whip->video_rtx_ssrc,
whip->video_ssrc,
whip->video_rtx_ssrc);
}

if (!av_bprint_is_complete(&bp)) {
Expand Down Expand Up @@ -894,6 +970,8 @@ static int parse_answer(AVFormatContext *s)
goto end;
}
}
} else if (av_strstart(line, "a=ice-lite", NULL)) {
whip->ice_lite_remote = 1;
}
}

Expand Down Expand Up @@ -985,6 +1063,22 @@ static int ice_create_request(AVFormatContext *s, uint8_t *buf, int buf_size, in
avio_wb16(pb, STUN_ATTR_USE_CANDIDATE); /* attribute type use-candidate */
avio_wb16(pb, 0); /* size of use-candidate */

/**
* For ICE-lite peers we are *always* the controlling agent (RFC 8445 6.1.3.1).
* Add PRIORITY + ICE-CONTROLLING attributes.
*/
if (whip->ice_lite_remote) {
/* we are controlling, use host-candidate priority 126 << 24 | 65535 << 8 | 255 = 2130706431 */
avio_wb16(pb, STUN_ATTR_PRIORITY);
avio_wb16(pb, 4);
avio_wb32(pb, 2130706431);

avio_wb16(pb, STUN_ATTR_ICE_CONTROLLING);
avio_wb16(pb, 8);
avio_wb32(pb, (uint32_t)(whip->ice_tie_breaker >> 32));
avio_wb32(pb, (uint32_t)(whip->ice_tie_breaker & 0xffffffff));
}

/* Build and update message integrity */
avio_wb16(pb, STUN_ATTR_MESSAGE_INTEGRITY); /* attribute type message integrity */
avio_wb16(pb, 20); /* size of message integrity */
Expand Down Expand Up @@ -1421,6 +1515,37 @@ static int setup_srtp(AVFormatContext *s)
return ret;
}


/**
* RTX history helpers
*/
static void rtp_history_store(WHIPContext *whip, const uint8_t *pkt, int size)
{
int pos = whip->hist_head % whip->history_size;
RtpHistoryItem * it = &whip->history[pos];
/* free older entry */
av_free(it->pkt);
it->pkt = av_malloc(size);
if (!it->pkt)
return;

memcpy(it->pkt, pkt, size);
it->size = size;
it->seq = AV_RB16(pkt + 2);

whip->hist_head++;
}

static const RtpHistoryItem* rtp_history_find(const WHIPContext *whip, uint16_t seq)
{
for (int i = 0; i < whip->history_size; i++) {
const RtpHistoryItem * it = &whip->history[i];
if (it->pkt && it->seq == seq)
return it;
}
return NULL;
}

/**
* Callback triggered by the RTP muxer when it creates and sends out an RTP packet.
*
Expand Down Expand Up @@ -1457,6 +1582,10 @@ static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size)
return 0;
}

/* Store only ORIGINAL video packets (non-RTX, non-RTCP) */
if (!is_rtcp && is_video)
rtp_history_store(whip, buf, buf_size);

ret = ffurl_write(whip->udp, whip->buf, cipher_size);
if (ret < 0) {
av_log(whip, AV_LOG_ERROR, "WHIP: Failed to write packet=%dB, ret=%d\n", cipher_size, ret);
Expand All @@ -1465,6 +1594,45 @@ static int on_rtp_write_packet(void *opaque, const uint8_t *buf, int buf_size)

return ret;
}
/**
* Build and send a single RTX packet
*/
static int send_rtx_packet(AVFormatContext *s, const uint8_t * orig_pkt, int orig_size)
{
WHIPContext * whip = s->priv_data;
int new_size, cipher_size;
/* skip if no RTX PT configured */
if (!whip->enable_nack_rtx)
return 0;

/* allocate new buffer: header + 2 + payload */
if (orig_size + 2 > sizeof(whip->buf))
return 0;

memcpy(whip->buf, orig_pkt, orig_size);

uint8_t * hdr = whip->buf;
uint16_t orig_seq = AV_RB16(hdr + 2);

/* rewrite header */
hdr[1] = (hdr[1] & 0x80) | whip->rtx_payload_type; /* keep M bit */
AV_WB16(hdr + 2, whip->rtx_seq++);
AV_WB32(hdr + 8, whip->video_rtx_ssrc);

/* shift payload 2 bytes */
memmove(hdr + 12 + 2, hdr + 12, orig_size - 12);
AV_WB16(hdr + 12, orig_seq);

new_size = orig_size + 2;

/* Encrypt by SRTP and send out. */
cipher_size = ff_srtp_encrypt(&whip->srtp_video_send, whip->buf, new_size, whip->buf, sizeof(whip->buf));
if (cipher_size <= 0 || cipher_size < new_size) {
av_log(whip, AV_LOG_WARNING, "WHIP: Failed to encrypt packet=%dB, cipher=%dB\n", new_size, cipher_size);
return 0;
}
return ffurl_write(whip->udp, whip->buf, cipher_size);
}

/**
* Creates dedicated RTP muxers for each stream in the AVFormatContext to build RTP
Expand Down Expand Up @@ -1775,24 +1943,75 @@ static int whip_write_packet(AVFormatContext *s, AVPacket *pkt)
AVStream *st = s->streams[pkt->stream_index];
AVFormatContext *rtp_ctx = st->priv_data;

/* TODO: Send binding request every 1s as WebRTC heartbeat. */
/* Periodic consent-freshness STUN Binding Request */
int64_t now = av_gettime();
if (now - whip->last_consent_tx >= (int64_t)whip->consent_interval * 1000) {
int req_sz;
if (ice_create_request(s, whip->buf, sizeof(whip->buf), &req_sz) >= 0 && ffurl_write(whip->udp, whip->buf, req_sz) == req_sz) {
whip->consent_failures++;
whip->last_consent_tx = now;
av_log(whip, AV_LOG_VERBOSE, "WHIP: consent-freshness request %d sent\n", whip->consent_failures);
}
}

/**
* Receive packets from the server such as ICE binding requests, DTLS messages,
* and RTCP like PLI requests, then respond to them.
*/
ret = ffurl_read(whip->udp, whip->buf, sizeof(whip->buf));
if (ret > 0) {
if (ice_is_binding_response(whip->buf, ret)) {
whip->consent_failures = 0;
av_log(whip, AV_LOG_VERBOSE, "WHIP: consent-freshness response received, counter reset\n");
}
if (is_dtls_packet(whip->buf, ret)) {
if ((ret = ffurl_write(whip->dtls_uc, whip->buf, ret)) < 0) {
av_log(whip, AV_LOG_ERROR, "WHIP: Failed to handle DTLS message\n");
goto end;
}
}
/* Handle RTCP NACK ( RTPFB / FMT=1 ) -------------- */
if (media_is_rtcp(whip->buf, ret)) {
int ptr = 0;
while (ptr + 4 <= ret) {
uint8_t pt = whip->buf[ptr + 1];
int len = (AV_RB16(&whip->buf[ptr + 2]) + 1) * 4;
if (ptr + len > ret) break;

if (pt == 205) { /* RTPFB */
uint8_t fmt = (whip->buf[ptr] & 0x1f);
if (fmt == 1 && len >= 12) {
uint16_t pid = AV_RB16(&whip->buf[ptr + 12 - 4]);
uint16_t blp = AV_RB16(&whip->buf[ptr + 14 - 4]);

/* retransmit pid + any bit set in blp */
for (int bit = -1; bit < 16; bit++) {
uint16_t seq = (bit < 0) ? pid : pid + bit + 1;
if (bit >= 0 && !(blp & (1 << bit)))
continue;

const RtpHistoryItem * it = rtp_history_find(whip, seq);
if (it)
send_rtx_packet(s, it->pkt, it->size);

}
}

}
ptr += len;
}

}
} else if (ret != AVERROR(EAGAIN)) {
av_log(whip, AV_LOG_ERROR, "WHIP: Failed to read from UDP socket\n");
goto end;
}
/* Check consent freshness consecutive failures */
if (whip->consent_failures >= WHIP_CONSENT_MAX_FAILURES) {
av_log(whip, AV_LOG_ERROR, "WHIP: No consent-freshness response after %d attempts, closing\n", WHIP_CONSENT_MAX_FAILURES);
ret = AVERROR(EHOSTUNREACH);
goto end;
}

if (whip->h264_annexb_insert_sps_pps && st->codecpar->codec_id == AV_CODEC_ID_H264) {
if ((ret = h264_annexb_insert_sps_pps(s, pkt)) < 0) {
Expand Down Expand Up @@ -1891,7 +2110,9 @@ static const AVOption options[] = {
{ "pkt_size", "The maximum size, in bytes, of RTP packets that send out", OFFSET(pkt_size), AV_OPT_TYPE_INT, { .i64 = 1200 }, -1, INT_MAX, DEC },
{ "authorization", "The optional Bearer token for WHIP Authorization", OFFSET(authorization), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC },
{ "cert_file", "The optional certificate file path for DTLS", OFFSET(cert_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC },
{ "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC },
{ "key_file", "The optional private key file path for DTLS", OFFSET(key_file), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, DEC },
{ "consent_interval", "STUN consent refresh interval in ms (RFC 7675)", OFFSET(consent_interval), AV_OPT_TYPE_INT, { .i64 = WHIP_CONSENT_DEF_INTERVAL }, 5000, 30000, DEC },
{ "rtx_history", "Packet history size", OFFSET(history_size), AV_OPT_TYPE_INT, { .i64 = HISTORY_SIZE_DEFAULT }, 64, 2048, DEC },
{ NULL },
};

Expand Down
Loading