Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FFmpeg: Add Whip Muxer support for subsecond latency streaming #1

Open
wants to merge 60 commits into
base: feature/whip
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
60 commits
Select commit Hold shift + click to select a range
ae07b06
WHIP: Add WebRTC WHIP muxer.
winlinvip Apr 16, 2023
f090d64
WHIP: Only support h264+opus codec.
winlinvip Apr 16, 2023
edaf213
WHIP: Generate and exchange offer server to get answer.
yangrtc Apr 21, 2023
83202f7
WHIP: Support parse ice from answer.
winlinvip Apr 22, 2023
8a63f74
WHIP: Refine code.
winlinvip Apr 22, 2023
67bccbf
WHIP: Support ICE handshake by binding.
winlinvip May 1, 2023
9db6961
WHIP: Support fast retransmit for binding request.
winlinvip May 1, 2023
1496684
WHIP: Update the dependency, requires openssl.
winlinvip May 2, 2023
0522db8
WHIP: Support DTLS handshake by openssl
winlinvip May 2, 2023
4ee4ca5
WHIP: Setup SRTP with key material generated by DTLS
winlinvip May 3, 2023
67f9942
WHIP: Refine code.
winlinvip May 3, 2023
ff5ef96
WHIP: Support mux and send audio by RTP
winlinvip May 3, 2023
98fca74
WHIP: Support mux and send video by RTP
winlinvip May 4, 2023
9ecbef0
WHIP: Support audio or video only
winlinvip May 4, 2023
40c385a
WHIP: Support video annexb format.
winlinvip May 4, 2023
fc0974a
WHIP: Refine code.
winlinvip May 5, 2023
fae7b40
WHIP: Dispose resource when write trailer.
winlinvip May 5, 2023
5e400cd
WHIP: Check alloc fail and return EMOMEM.
winlinvip May 5, 2023
d9c7dc7
WHIP: Always dispose resource by WHIP.
winlinvip May 5, 2023
235aaa0
WHIP: Refine macros.
winlinvip May 6, 2023
34fd9f9
WHIP: Support set parameters by options.
winlinvip May 6, 2023
2bfdb67
WHIP: Eliminate the unused write_trailer.
winlinvip May 6, 2023
e3fdedd
WHIP: Refine code.
winlinvip May 6, 2023
b4574d6
WHIP: Check for null string to avoid strlen crash, scanned by ASAN
winlinvip May 11, 2023
248243a
WHIP: Improve macro and options comments with more background details
winlinvip May 11, 2023
d801061
WHIP: Remote password should be used to generate the message integrity.
winlinvip May 12, 2023
a1a8335
Use AVPrint to generate the offer and answer for WebRTC SDP.
winlinvip May 12, 2023
ff72920
WHIP:Support send sctp
duiniuluantanqin May 12, 2023
fc3a4a2
WHIP: Refining SRTP context selection.
winlinvip May 12, 2023
45e6ece
WHIP: Refine code.
winlinvip May 13, 2023
6f965d6
Add bidirectional ICE binding, DTLS with ECDSA, and compatibility imp…
cloudwebrtc May 13, 2023
3fd6767
Add missing quotation string to error message
winlinvip May 15, 2023
68ef7ad
WHIP: Extract DTLSContext from RTContext.
winlinvip May 17, 2023
c7b146b
WHIP: Reorder functions, nothing changed.
winlinvip May 17, 2023
2dedc86
WHIP: Remove the macro that suppresses warnings for the SSL deprecate…
winlinvip May 18, 2023
1054bed
WHIP: Merge write header to init.
winlinvip May 18, 2023
b31e0c0
WHIP:Support baseline/main/high profile without B frames (#2)
duiniuluantanqin May 19, 2023
b3f5c27
WHIP: Fix the SSL deprecated warning by replacing EC_KEY_new with EVP…
winlinvip May 23, 2023
85c5680
WHIP: Refine the code to be shorter.
winlinvip May 23, 2023
71ee877
WHIP: Increase the base timeout and thereby reduce the number of unne…
winlinvip May 23, 2023
726bb67
WHIP: Refine ARQ for DTLS with bug fixed.
winlinvip May 23, 2023
6e1fbb5
WHIP: Update muxers.texi for RTC.
winlinvip May 29, 2023
86b361c
WHIP: Fix bugs causing build OpenSSL error, to work with Pion
winlinvip May 30, 2023
ea74e3c
Optimize DTLS Handshake and ICE Handling for Improved Performance (#3)
winlinvip Jun 7, 2023
680ed49
Roundtrip3: Refinements Based on Derek Buitenhuis' Comments (#4)
winlinvip Jun 9, 2023
2ac4100
WHIP: Do not hardcode codec for SDP. (#5)
winlinvip Jun 9, 2023
5819297
WHIP: Parse profile and level from extradata.
winlinvip Jun 9, 2023
0c4a158
WHIP: Insert SPS and PPS before IDR frames in annexb format due to h2…
winlinvip Jun 10, 2023
812932e
WHIP: Failed immediately if there is no profile and extradata present.
winlinvip Jun 10, 2023
aaf26ef
WHIP: Refine names and comments.
winlinvip Jun 19, 2023
66a064f
WHIP: Free buffer leak of pb for RTP muxer.
winlinvip Jun 20, 2023
9c7a091
WHIP: Use options for passing parameters to the HTTP or UDP context.
winlinvip Jun 21, 2023
de33fcc
WHIP: Enhance security by using BearToken for delete API.
winlinvip Jun 21, 2023
8971a1f
WHIP: Support user specified certificate and key file.
winlinvip Jun 21, 2023
0bd6867
WHIP: Read DTLS certificate file by ffurl.
winlinvip Jul 4, 2023
0b95312
WHIP: Free bio_in if ENOMEM.
winlinvip Jul 8, 2023
63e3a55
WHIP: Refine DTLS, extract DTLS APIs.
winlinvip Aug 24, 2023
172d34b
Sync with FFmpeg master 5ddab49d48
winlinvip Oct 17, 2023
d3561e6
Extract DTLS to dtls.c
winlinvip Oct 17, 2023
3b15ba1
Change some important step log level to info.
winlinvip Oct 17, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
WHIP: Refine the code to be shorter.
  • Loading branch information
winlinvip committed Oct 17, 2023
commit 85c5680af7d1eb8f3a38622ab1df1cadf3c28e53
141 changes: 58 additions & 83 deletions libavformat/rtcenc.c
Original file line number Diff line number Diff line change
Expand Up @@ -23,39 +23,29 @@

#ifndef CONFIG_OPENSSL
#error "DTLS is not supported, please enable openssl"
#endif

#if CONFIG_OPENSSL
#else
#include <openssl/ssl.h>
#include <openssl/err.h>
#if OPENSSL_VERSION_NUMBER < 0x1010102fL
#error "OpenSSL version 1.1.1b or newer is required"
#endif
#endif

#include "libavutil/dict.h"
#include "libavutil/avassert.h"
#include "libavutil/mathematics.h"
#include "libavcodec/codec_desc.h"
#include "libavcodec/mpeg4audio.h"
#include "avformat.h"
#include "internal.h"
#include "mux.h"
#include "libavutil/opt.h"
#include "libavcodec/avcodec.h"
#include "libavutil/avstring.h"
#include "url.h"
#include "libavutil/random_seed.h"
#include "avio_internal.h"
#include "libavutil/hmac.h"
#include "libavutil/base64.h"
#include "libavutil/bprint.h"
#include "libavutil/crc.h"
#include "network.h"
#include "libavutil/hmac.h"
#include "libavutil/opt.h"
#include "libavutil/random_seed.h"
#include "libavutil/time.h"
#include "libavutil/base64.h"
#include "srtp.h"
#include "avc.h"
#include "avio_internal.h"
#include "http.h"
#include "libavutil/bprint.h"
#include "internal.h"
#include "mux.h"
#include "network.h"
#include "srtp.h"

/**
* Maximum size limit of a Session Description Protocol (SDP),
Expand Down Expand Up @@ -221,6 +211,10 @@ static av_cold int dtls_context_init(DTLSContext *ctx)

/* Generate a self-signed certificate. */
subject = X509_NAME_new();
if (!subject) {
ret = AVERROR(ENOMEM);
goto end;
}

serial = (int)av_get_random_seed();
if (ASN1_INTEGER_set(X509_get_serialNumber(dtls_cert), serial) != 1) {
Expand Down Expand Up @@ -328,18 +322,15 @@ static av_cold void dtls_context_deinit(DTLSContext *ctx)
static void openssl_on_info(const SSL *dtls, int where, int ret)
{
int w, r1;
const char *method, *alert_type, *alert_desc;
const char *method = "undefined", *alert_type, *alert_desc;
DTLSContext *ctx = (DTLSContext*)SSL_get_ex_data(dtls, 0);
void *s1 = ctx->log_avcl;

w = where & ~SSL_ST_MASK;
if (w & SSL_ST_CONNECT) {
if (w & SSL_ST_CONNECT)
method = "SSL_connect";
} else if (w & SSL_ST_ACCEPT) {
else if (w & SSL_ST_ACCEPT)
method = "SSL_accept";
} else {
method = "undefined";
}

r1 = SSL_get_error(dtls, ret);
if (where & SSL_CB_LOOP) {
Expand All @@ -351,26 +342,23 @@ static void openssl_on_info(const SSL *dtls, int where, int ret)
alert_type = SSL_alert_type_string_long(ret);
alert_desc = SSL_alert_desc_string(ret);

if (!av_strcasecmp(alert_type, "warning") && !av_strcasecmp(alert_desc, "CN")) {
if (!av_strcasecmp(alert_type, "warning") && !av_strcasecmp(alert_desc, "CN"))
av_log(s1, AV_LOG_WARNING, "DTLS: SSL3 alert method=%s type=%s, desc=%s(%s), where=%d, ret=%d, r1=%d\n",
method, alert_type, alert_desc, SSL_alert_desc_string_long(ret), where, ret, r1);
} else {
else
av_log(s1, AV_LOG_ERROR, "DTLS: SSL3 alert method=%s type=%s, desc=%s(%s), where=%d, ret=%d, r1=%d\n",
method, alert_type, alert_desc, SSL_alert_desc_string_long(ret), where, ret, r1);
}
} else if (where & SSL_CB_EXIT) {
if (!ret) {
if (!ret)
av_log(s1, AV_LOG_WARNING, "DTLS: Fail method=%s state=%s(%s), where=%d, ret=%d, r1=%d\n",
method, SSL_state_string(dtls), SSL_state_string_long(dtls), where, ret, r1);
} else if (ret < 0) {
if (r1 != SSL_ERROR_NONE && r1 != SSL_ERROR_WANT_READ && r1 != SSL_ERROR_WANT_WRITE) {
else if (ret < 0)
if (r1 != SSL_ERROR_NONE && r1 != SSL_ERROR_WANT_READ && r1 != SSL_ERROR_WANT_WRITE)
av_log(s1, AV_LOG_ERROR, "DTLS: Error method=%s state=%s(%s), where=%d, ret=%d, r1=%d\n",
method, SSL_state_string(dtls), SSL_state_string_long(dtls), where, ret, r1);
} else {
else
av_log(s1, AV_LOG_VERBOSE, "DTLS: method=%s state=%s(%s), where=%d, ret=%d, r1=%d\n",
method, SSL_state_string(dtls), SSL_state_string_long(dtls), where, ret, r1);
}
}
}
}

Expand All @@ -384,9 +372,8 @@ static unsigned int openssl_dtls_timer_cb(SSL *dtls, unsigned int previous_us)

/* If previous_us is 0, for example, the HelloVerifyRequest, we should respond it ASAP.
* when got ServerHello, we should reset the timer. */
if (!previous_us || ctx->dtls_should_reset_timer) {
if (!previous_us || ctx->dtls_should_reset_timer)
timeout_us = ctx->dtls_arq_timeout * 1000; /* in us */
}

/* never exceed the max timeout. */
timeout_us = FFMIN(timeout_us, 30 * 1000 * 1000); /* in us */
Expand All @@ -405,38 +392,18 @@ static void openssl_state_trace(DTLSContext *ctx, uint8_t *data, int length, int
void *s1 = ctx->log_avcl;

/* Change_cipher_spec(20), alert(21), handshake(22), application_data(23) */
if (length >= 1) {
if (length >= 1)
content_type = (uint8_t)data[0];
}

if (length >= 13) {
if (length >= 13)
size = (uint16_t)(data[11])<<8 | (uint16_t)data[12];
}

if (length >= 14) {
if (length >= 14)
handshake_type = (uint8_t)data[13];
}

av_log(s1, AV_LOG_VERBOSE, "WHIP: DTLS state %s %s, done=%u, arq=%u, r0=%d, r1=%d, len=%u, cnt=%u, size=%u, hs=%u\n",
"Active", (incoming? "RECV":"SEND"), ctx->dtls_done_for_us, ctx->dtls_arq_packets, r0, r1, length,
content_type, size, handshake_type);
}

/**
* The return value of verify_callback controls the strategy of the further verification process. If verify_callback
* returns 0, the verification process is immediately stopped with "verification failed" state. If SSL_VERIFY_PEER is
* set, a verification failure alert is sent to the peer and the TLS/SSL handshake is terminated. If verify_callback
* returns 1, the verification process is continued. If verify_callback always returns 1, the TLS/SSL handshake will
* not be terminated with respect to verification failures and the connection will be established. The calling process
* can however retrieve the error code of the last verification error using SSL_get_verify_result(3) or by maintaining
* its own error storage managed by verify_callback.
*/
static int openssl_verify_callback(int preverify_ok, X509_STORE_CTX *ctx)
{
/* Always OK, we don't check the certificate of client, because we allow client self-sign certificate. */
return 1;
}

/**
* Initializes DTLS context for client role using ECDHE.
*/
Expand Down Expand Up @@ -471,8 +438,6 @@ static av_cold int openssl_init_dtls_context(DTLSContext *ctx, SSL_CTX *dtls_ctx
ret = AVERROR(EINVAL);
goto end;
}
/* Server will send Certificate Request. */
SSL_CTX_set_verify(dtls_ctx, SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE, openssl_verify_callback);
/* The depth count is "level 0:peer certificate", "level 1: CA certificate",
* "level 2: higher level CA certificate", and so on. */
SSL_CTX_set_verify_depth(dtls_ctx, 4);
Expand Down Expand Up @@ -636,6 +601,10 @@ static int dtls_context_handshake(DTLSContext *ctx)
void *s1 = ctx->log_avcl;

dtls_ctx = SSL_CTX_new(DTLS_client_method());
if (!dtls_ctx) {
ret = AVERROR(ENOMEM);
goto end;
}

if (!ctx->udp_uc) {
av_log(s1, AV_LOG_ERROR, "DTLS: No UDP context\n");
Expand All @@ -651,9 +620,23 @@ static int dtls_context_handshake(DTLSContext *ctx)

/* The dtls should not be created unless the dtls_ctx has been initialized. */
dtls = SSL_new(dtls_ctx);
if (!dtls) {
ret = AVERROR(ENOMEM);
goto end;
}

bio_in = BIO_new(BIO_s_mem());
if (!bio_in) {
ret = AVERROR(ENOMEM);
goto end;
}

bio_out = BIO_new(BIO_s_mem());
if (!bio_out) {
ret = AVERROR(ENOMEM);
goto end;
}

SSL_set_bio(dtls, bio_in, bio_out);

ret = openssl_init_dtls_ssl(ctx, dtls);
Expand Down Expand Up @@ -794,10 +777,9 @@ static av_cold int whip_init(AVFormatContext *s)
av_log(s, AV_LOG_INFO, "WHIP: Init ice_arq_max=%d, ice_arq_timeout=%d, dtls_arq_max=%d, dtls_arq_timeout=%d pkt_size=%d\n",
rtc->ice_arq_max, rtc->ice_arq_timeout, rtc->dtls_arq_max, rtc->dtls_arq_timeout, rtc->pkt_size);

if (rtc->pkt_size < ideal_pkt_size) {
if (rtc->pkt_size < ideal_pkt_size)
av_log(s, AV_LOG_WARNING, "WHIP: pkt_size=%d(<%d) is too small, may cause packet loss\n",
rtc->pkt_size, ideal_pkt_size);
}

return 0;
}
Expand Down Expand Up @@ -1682,11 +1664,8 @@ static int ice_handshake(AVFormatContext *s)
break;

/* When a binding request is received, it is necessary to respond immediately. */
if (ice_is_binding_request(res_buf, ret)) {
if ((ret = ice_handle_binding_request(s, res_buf, ret)) < 0) {
goto end;
}
}
if (ice_is_binding_request(res_buf, ret) && (ret = ice_handle_binding_request(s, res_buf, ret)) < 0)
goto end;
}

/* Wait just for a small while to get the possible binding request from server. */
Expand All @@ -1711,15 +1690,15 @@ static int ice_handshake(AVFormatContext *s)

/* When a binding request is received, it is necessary to respond immediately. */
if (ice_is_binding_request(res_buf, ret)) {
if ((ret = ice_handle_binding_request(s, res_buf, ret)) < 0) {
if ((ret = ice_handle_binding_request(s, res_buf, ret)) < 0)
goto end;
}
break;
}
}

av_log(s, AV_LOG_INFO, "WHIP: ICE STUN ok, url=udp://%s:%d, username=%s:%s, req=%dB, res=%dB, arq=%d\n",
rtc->ice_host, rtc->ice_port, rtc->ice_ufrag_remote, rtc->ice_ufrag_local, size, ret,
av_log(s, AV_LOG_INFO, "WHIP: ICE STUN ok, url=udp://%s:%d, location=%s, username=%s:%s, req=%dB, res=%dB, arq=%d\n",
rtc->ice_host, rtc->ice_port, rtc->whip_resource_url ? rtc->whip_resource_url : "",
rtc->ice_ufrag_remote, rtc->ice_ufrag_local, size, ret,
rtc->ice_arq_max - fast_retries);
ret = 0;

Expand Down Expand Up @@ -1922,9 +1901,8 @@ static int on_rtp_write_packet(void *opaque, uint8_t *buf, int buf_size)
is_rtcp = buf[1] >= 192 && buf[1] <= 223;
payload_type = buf[1] & 0x7f;
is_video = payload_type == rtc->video_payload_type;
if (!is_rtcp && payload_type != rtc->video_payload_type && payload_type != rtc->audio_payload_type) {
if (!is_rtcp && payload_type != rtc->video_payload_type && payload_type != rtc->audio_payload_type)
return 0;
}

/**
* For video, the STAP-A with SPS/PPS should:
Expand All @@ -1935,14 +1913,12 @@ static int on_rtp_write_packet(void *opaque, uint8_t *buf, int buf_size)
nalu_header = buf[12] & 0x1f;
if (nalu_header == NALU_TYPE_STAP_A) {
/* Reset the marker bit to 0. */
if (buf[1] & 0x80) {
if (buf[1] & 0x80)
buf[1] &= 0x7f;
}

/* Reset the NRI to the first NALU's NRI. */
if (buf_size > 15 && (buf[15]&0x60) != (buf[12]&0x60)) {
if (buf_size > 15 && (buf[15]&0x60) != (buf[12]&0x60))
buf[12] = (buf[12]&0x80) | (buf[15]&0x60) | (buf[12]&0x1f);
}
}
}

Expand Down Expand Up @@ -2076,7 +2052,7 @@ static int whip_dispose(AVFormatContext *s)
}
}

av_log(s, AV_LOG_INFO, "WHIP: Dispose resource %s\n", rtc->whip_resource_url);
av_log(s, AV_LOG_INFO, "WHIP: Dispose resource %s ok\n", rtc->whip_resource_url);

end:
ffurl_closep(&whip_uc);
Expand Down Expand Up @@ -2149,9 +2125,8 @@ static int rtc_write_packet(AVFormatContext *s, AVPacket *pkt)
if (ret == AVERROR(EINVAL)) {
av_log(s, AV_LOG_WARNING, "Ignore failed to write packet=%dB, ret=%d\n", pkt->size, ret);
ret = 0;
} else {
} else
av_log(s, AV_LOG_ERROR, "Failed to write packet, size=%d\n", pkt->size);
}
return ret;
}

Expand Down