diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a977916 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vagrant/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..f85eb92 --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +This is a HTTP Live Streaming (HLS) server based on the nginx-rtmp-module, ffmpeg and the html video element. + +[HTTP Live Streaming](https://en.wikipedia.org/wiki/HTTP_Live_Streaming) (HLS) uses the [MPEG-2 Transport Stream](https://en.wikipedia.org/wiki/MPEG_transport_stream) (MP2T) to transport [H.264](https://en.wikipedia.org/wiki/H.264/MPEG-4_AVC) video and [AAC](https://en.wikipedia.org/wiki/Advanced_Audio_Coding)/[MP3](https://en.wikipedia.org/wiki/MP3) audio. On the browser, via JavaScript, MP2T is transmuxed into the [ISO BMFF](https://en.wikipedia.org/wiki/ISO_base_media_file_format) Byte Stream Format and feed to the html video element via [Media Source Extensions](https://en.wikipedia.org/wiki/Media_Source_Extensions) (MSE). + +# Usage + +Install the [Ubuntu Base Box](https://github.com/rgl/ubuntu-vagrant). + +Run `vagrant up` to launch with VirtualBox. + +Browse to [http://10.0.0.2/](http://10.0.0.2/) to see the examples. + +# Reference + +* [Setting up HLS live streaming server using NGINX + nginx-rtmp-module on Ubuntu](https://docs.peer5.com/guides/setting-up-hls-live-streaming-server-using-nginx/) +* [FFmpeg and H.264 Encoding Guide](https://trac.ffmpeg.org/wiki/Encode/H.264) +* [Apple HTTP Live Streaming](https://developer.apple.com/streaming/) +* [Apple HLS Authoring Specification: General Authoring Requirements](https://developer.apple.com/library/content/documentation/General/Reference/HLSAuthoringSpec/Requirements.html) +* [Apple HTTP Live Streaming Examples](https://developer.apple.com/streaming/examples/) +* [RFC8216: HTTP Live Streaming](https://tools.ietf.org/html/rfc8216) diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..20d6f3a --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,17 @@ +Vagrant.configure(2) do |config| + config.vm.box = 'ubuntu-16.04-amd64' + + config.vm.hostname = 'streaming' + + config.vm.provider 'virtualbox' do |vb| + vb.linked_clone = true + vb.memory = 3072 + vb.cpus = 2 + end + + config.vm.network "private_network", ip: "10.0.0.2" + + config.vm.provision 'shell', path: 'provision-base.sh' + config.vm.provision 'shell', path: 'provision-nginx-rtmp-module.sh' + config.vm.provision 'shell', path: 'provision-videos.sh' +end diff --git a/config/etc/inputrc b/config/etc/inputrc new file mode 100644 index 0000000..0c3aced --- /dev/null +++ b/config/etc/inputrc @@ -0,0 +1,8 @@ +set input-meta on +set output-meta on +set show-all-if-ambiguous on +set completion-ignore-case on +"\e[A": history-search-backward +"\e[B": history-search-forward +"\eOD": backward-word +"\eOC": forward-word diff --git a/config/etc/profile.d/login.sh b/config/etc/profile.d/login.sh new file mode 100644 index 0000000..c83713a --- /dev/null +++ b/config/etc/profile.d/login.sh @@ -0,0 +1,7 @@ +[[ "$-" != *i* ]] && return +export EDITOR=vim +export PAGER=less +alias l='ls -lF --color' +alias ll='l -a' +alias h='history 25' +alias j='jobs -l' diff --git a/config/etc/vim/vimrc.local b/config/etc/vim/vimrc.local new file mode 100644 index 0000000..31cc1da --- /dev/null +++ b/config/etc/vim/vimrc.local @@ -0,0 +1,23 @@ +syntax on +set background=dark +set esckeys +set ruler +set laststatus=2 +set nobackup + +autocmd BufNewFile,BufRead Vagrantfile set ft=ruby +autocmd BufNewFile,BufRead *.config set ft=xml + +" Usefull setting for working with Ruby files. +autocmd FileType ruby set tabstop=2 shiftwidth=2 smarttab expandtab softtabstop=2 autoindent +autocmd FileType ruby set smartindent cinwords=if,elsif,else,for,while,try,rescue,ensure,def,class,module + +" Usefull setting for working with Python files. +autocmd FileType python set tabstop=4 shiftwidth=4 smarttab expandtab softtabstop=4 autoindent +" Automatically indent a line that starts with the following words (after we press ENTER). +autocmd FileType python set smartindent cinwords=if,elif,else,for,while,try,except,finally,def,class + +" Usefull setting for working with Go files. +autocmd FileType go set tabstop=4 shiftwidth=4 smarttab expandtab softtabstop=4 autoindent +" Automatically indent a line that starts with the following words (after we press ENTER). +autocmd FileType go set smartindent cinwords=if,else,switch,for,func diff --git a/provision-base.sh b/provision-base.sh new file mode 100644 index 0000000..8fd764b --- /dev/null +++ b/provision-base.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# abort this script on errors. +set -eux + +# prevent apt-get et al from opening stdin. +# NB even with this, you'll still get some warnings that you can ignore: +# dpkg-preconfigure: unable to re-open stdin: No such file or directory +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get install -y git-core +apt-get install -y unzip xz-utils +apt-get install -y --no-install-recommends httpie +apt-get install -y --no-install-recommends vim +apt-get install -y --no-install-recommends jq + +# set system configuration. +rm -f /{root,home/*}/.{profile,bashrc} +cp -v -r /vagrant/config/etc/* /etc + +su vagrant -c bash <<'VAGRANT_EOF' +#!/bin/bash +# abort this script on errors. +set -eux + +# configure git. +# see http://stackoverflow.com/a/12492094/477532 +git config --global user.name 'Rui Lopes' +git config --global user.email 'rgl@ruilopes.com' +git config --global push.default simple +#git config --list --show-origin +VAGRANT_EOF + +apt-get autoremove -y --purge diff --git a/provision-nginx-rtmp-module.sh b/provision-nginx-rtmp-module.sh new file mode 100644 index 0000000..ab971da --- /dev/null +++ b/provision-nginx-rtmp-module.sh @@ -0,0 +1,154 @@ +#!/bin/bash +set -eux + +# add the nginx user. +groupadd --system nginx-rtmp +adduser \ + --system \ + --disabled-login \ + --no-create-home \ + --gecos '' \ + --ingroup nginx-rtmp \ + --home /opt/nginx-rtmp \ + nginx-rtmp +install -d -o root -g root -m 755 /opt/nginx-rtmp +install -d -o root -g root -m 755 /opt/nginx-rtmp/public + +# download and install the latest version of ffmpeg. +wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-64bit-static.tar.xz +tar xf ffmpeg-release-64bit-static.tar.xz +cp ffmpeg-*-static/{ffmpeg,ffprobe} /usr/local/bin +ffmpeg -version + +# download the latest version of nginx-rtmp-module. +git clone https://github.com/sergey-dryabzhinsky/nginx-rtmp-module.git + +# download, build and install nginx+nginx-rtmp-module. +wget -q https://nginx.org/download/nginx-1.13.4.tar.gz +tar xf nginx-1.13.4.tar.gz +pushd nginx-1.13.4 +apt-get install -y libpcre3 libpcre3-dev libssl-dev +./configure \ + --prefix=/opt/nginx-rtmp \ + --build=nginx-rtmp \ + --user=nginx-rtmp \ + --group=nginx-rtmp \ + --add-module=../nginx-rtmp-module +make -j 2 +make install #DESTDIR=$PWD/DIST +popd + +# copy public data. +cp -r /vagrant/public /opt/nginx-rtmp +wget -qO /opt/nginx-rtmp/public/shaka-player.compiled.js https://cdnjs.cloudflare.com/ajax/libs/shaka-player/2.2.0/shaka-player.compiled.js +wget -qO /opt/nginx-rtmp/public/hls.light.js https://github.com/video-dev/hls.js/raw/master/dist/hls.light.js +wget -qO /opt/nginx-rtmp/public/hls.light.min.js https://github.com/video-dev/hls.js/raw/master/dist/hls.light.min.js +wget -q https://github.com/videojs/video.js/releases/download/v6.2.7/video-js-6.2.7.zip +unzip -d video-js-6.2.7 video-js-6.2.7.zip +cp video-js-6.2.7/{video{,.min}.js,video-js{,.min}.css} /opt/nginx-rtmp/public +wget -qO /opt/nginx-rtmp/public/videojs-contrib-hls.js https://github.com/videojs/videojs-contrib-hls/releases/download/v5.10.0/videojs-contrib-hls.js +wget -qO /opt/nginx-rtmp/public/videojs-contrib-hls.min.js https://github.com/videojs/videojs-contrib-hls/releases/download/v5.10.0/videojs-contrib-hls.min.js +cp nginx-rtmp-module/stat.xsl /opt/nginx-rtmp/public + +# create a tiny tmpfs for storing the video fragments. +cat >>/etc/fstab </opt/nginx-rtmp/conf/nginx.conf <<'EOF' +#error_log stderr warn; + +worker_processes auto; +events { + worker_connections 1024; +} + +rtmp { + server { + listen 1935; + application hls { + live on; + hls on; + hls_nested on; + hls_fragment 3s; + hls_playlist_length 3m; + hls_path /opt/nginx-rtmp/fragments/hls; + #allow publish 127.0.0.1; + #deny publish all; + #deny play all; + } + } +} + +http { + sendfile on; + tcp_nopush on; + root /opt/nginx-rtmp/public; + + types { + text/html html; + text/css css; + text/javascript js; + text/xsl xsl; + application/dash+xml mpd; + application/vnd.apple.mpegurl m3u8; + video/mp2t ts; + } + default_type application/octet-stream; + + server { + listen 80; + + location = /stat { + rtmp_stat all; + rtmp_stat_stylesheet stat.xsl; + } + + location /hls/ { + #add_header Cache-Control no-cache; + root /opt/nginx-rtmp/fragments; + } + } +} +EOF +/opt/nginx-rtmp/sbin/nginx -t + +# run as a service. +cat >/etc/systemd/system/nginx-rtmp.service <<'EOF' +[Unit] +Description=nginx-rtmp +After=network.target + +[Service] +Type=simple +ExecStart=/opt/nginx-rtmp/sbin/nginx -g 'daemon off;' +Restart=always + +[Install] +WantedBy=multi-user.target +EOF + +# start nginx. +systemctl enable nginx-rtmp +systemctl start nginx-rtmp + +# configure log rotation. +cat >/etc/logrotate.d/nginx-rtmp <<'EOF' +/opt/nginx-rtmp/logs/*.log { + daily + missingok + rotate 14 + compress + delaycompress + notifempty + create 0640 root root + sharedscripts + postrotate + systemctl kill --signal=USR1 --kill-who=main nginx-rtmp + endscript +} +EOF diff --git a/provision-videos.sh b/provision-videos.sh new file mode 100644 index 0000000..73319c2 --- /dev/null +++ b/provision-videos.sh @@ -0,0 +1,125 @@ + +apt-get install -y fonts-dejavu-core + +wget -q -O /usr/local/bin/youtube-dl https://yt-dl.org/downloads/latest/youtube-dl +chmod +x /usr/local/bin/youtube-dl + +wget -q -O /usr/local/bin/iframe-probe.py https://gist.githubusercontent.com/use-sparingly/7041ee993adb5c911f90/raw/d6cfe6a51c990ff5ae5242cb5711d2a68651f573/iframe-probe.py +chmod +x /usr/local/bin/iframe-probe.py + +mkdir videos +cd videos + +convert_video() { + input=$1; shift + output=$1; shift + max_video_height=$1; shift + fragment_length_seconds=$1; shift + fps=$1; shift + gop=$(expr $fragment_length_seconds \* $fps) + + echo "Transcoding $input into $output (${max_video_height}p)..." + # transcode (and scale) the original file to be directly consumed by the + # nginx-rtmp-module. + # NB the players are quite fragile, so its safer to use a constant GOP + # length, no scene detection (bc it causes the keyframe interval to + # vary) and closed GOPs. + # see https://trac.ffmpeg.org/wiki/Scaling%20(resizing)%20with%20ffmpeg + # see https://trac.ffmpeg.org/wiki/Encode/H.264 + # see https://kvssoft.wordpress.com/2015/01/28/mpeg-dash-gop/ + # see http://caniuse.com/#feat=mpeg4 + extra_filter_v="drawtext=text='%{pts\\:hms} #%{n}':x=-5:y=3:fontsize=13:fontcolor=white:box=1:boxborderw=3:boxcolor=black:fontfile=/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf" + ffmpeg \ + -loglevel info \ + -y \ + -i $input \ + -r $fps \ + -codec:v libx264 \ + -preset medium \ + -profile:v high -level 4.2 \ + -crf 23 \ + -g $gop \ + -keyint_min $gop \ + -sc_threshold 0 \ + -flags +cgop \ + -movflags +faststart \ + -filter:v "scale=w=-2:h=$max_video_height,$extra_filter_v" \ + -codec:a aac \ + -q:a 4 \ + -f flv \ + -vstats_file ${output}_${max_video_height}p_stats.txt \ + ${output}_${max_video_height}p.flv \ + 2>&1 \ + | grep -vEe 'Past duration [0-9.]+ too large' \ + | grep -vEe ' dropping frame ' + + echo "Dumping GOPs..." + # NB this calls ffprobe -show_frames -print_format json costa_rica_${max_video_height}p.flv + iframe-probe.py ${output}_${max_video_height}p.flv >${output}_${max_video_height}p_gops.txt + printf " GOPs size\ttype\n"; awk '{print $3 "\t" $4}' ${output}_${max_video_height}p_gops.txt | sort | uniq -c + + echo "Converting to static hls at /opt/nginx-rtmp/public/vod/hls/${output}/index.m3u8..." + rm -rf /opt/nginx-rtmp/public/vod/hls/${output} + mkdir -p /opt/nginx-rtmp/public/vod/hls/${output} + ffmpeg \ + -loglevel info \ + -i ${output}_${max_video_height}p.flv \ + -codec:v copy \ + -codec:a copy \ + -hls_time $fragment_length_seconds \ + -hls_list_size 0 \ + -f hls \ + /opt/nginx-rtmp/public/vod/hls/${output}/index.m3u8 +} + +echo 'Downloading the Costa Rica video....' +youtube-dl -o costa_rica_720p.webm -f 'best[height=720]' 'https://www.youtube.com/watch?v=iNJdPyoqt8U' +convert_video costa_rica_720p.webm costa_rica 240 3 24 + +echo 'Downloading the Kung Fu Mantis vs Jumping Spider video....' +youtube-dl -o kung_fu_mantis_vs_jumping_spider_720p.webm -f 'best[height=720]' 'https://www.youtube.com/watch?v=7wKu13wmHog' +convert_video kung_fu_mantis_vs_jumping_spider_720p.webm kung_fu_mantis_vs_jumping_spider 240 3 24 + +echo 'Downloading the Planet Earth II Continues video....' +youtube-dl -o planet_earth_ii_continues_trailer_720p.webm -f 'best[height=720]' 'https://www.youtube.com/watch?v=h8yo_Sp-rGY' +convert_video planet_earth_ii_continues_trailer_720p.webm planet_earth_ii_continues_trailer 240 3 24 + +echo 'Downloading the Tears of Steel video....' +wget -q http://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_720p.mov +convert_video tears_of_steel_720p.mov tears_of_steel 240 3 24 + +# continually stream the videos to nginx-rtmp in a background service. +cat >stream-from-files.sh <<'EOF' +#!/bin/bash +set -eux +while true; do + for n in *_240p.flv; do + ffmpeg \ + -loglevel info \ + -re \ + -i $n \ + -codec:v copy \ + -codec:a copy \ + -f flv \ + rtmp://localhost:1935/hls/live + sleep 1 + done +done +EOF +chmod +x stream-from-files.sh +cat >/etc/systemd/system/stream-from-files.service < + + + + streaming in nginx-rtmp-module + + +

+ nginx-rtmp-module stats +

+

live

+ +

static

+ + + \ No newline at end of file diff --git a/public/live/hlsjs.html b/public/live/hlsjs.html new file mode 100644 index 0000000..4253c80 --- /dev/null +++ b/public/live/hlsjs.html @@ -0,0 +1,23 @@ + + + + + live hls in hls.js + + + + + + + \ No newline at end of file diff --git a/public/live/videojs.html b/public/live/videojs.html new file mode 100644 index 0000000..fd4035f --- /dev/null +++ b/public/live/videojs.html @@ -0,0 +1,21 @@ + + + + + live hls in video.js + + + + + + + + + \ No newline at end of file diff --git a/public/static/hlsjs.html b/public/static/hlsjs.html new file mode 100644 index 0000000..571843a --- /dev/null +++ b/public/static/hlsjs.html @@ -0,0 +1,23 @@ + + + + + static hls in hls.js + + + + + + + \ No newline at end of file diff --git a/public/static/videojs.html b/public/static/videojs.html new file mode 100644 index 0000000..6b3d6e5 --- /dev/null +++ b/public/static/videojs.html @@ -0,0 +1,21 @@ + + + + + static hls in video.js + + + + + + + + + \ No newline at end of file