init

.gitignore

@@ -0,0 +1 @@
+.vagrant/

README.md

@@ -0,0 +1,20 @@
+This is a HTTP Live Streaming (HLS) server based on the nginx-rtmp-module, ffmpeg and the html video element.
+
+[HTTP Live Streaming](https://en.wikipedia.org/wiki/HTTP_Live_Streaming) (HLS) uses the [MPEG-2 Transport Stream](https://en.wikipedia.org/wiki/MPEG_transport_stream) (MP2T) to transport [H.264](https://en.wikipedia.org/wiki/H.264/MPEG-4_AVC) video and [AAC](https://en.wikipedia.org/wiki/Advanced_Audio_Coding)/[MP3](https://en.wikipedia.org/wiki/MP3) audio. On the browser, via JavaScript, MP2T is transmuxed into the [ISO BMFF](https://en.wikipedia.org/wiki/ISO_base_media_file_format) Byte Stream Format and feed to the html video element via [Media Source Extensions](https://en.wikipedia.org/wiki/Media_Source_Extensions) (MSE).
+
+# Usage
+
+Install the [Ubuntu Base Box](https://github.com/rgl/ubuntu-vagrant).
+
+Run `vagrant up` to launch with VirtualBox.
+
+Browse to [http://10.0.0.2/](http://10.0.0.2/) to see the examples.
+
+# Reference
+
+* [Setting up HLS live streaming server using NGINX + nginx-rtmp-module on Ubuntu](https://docs.peer5.com/guides/setting-up-hls-live-streaming-server-using-nginx/)
+* [FFmpeg and H.264 Encoding Guide](https://trac.ffmpeg.org/wiki/Encode/H.264)
+* [Apple HTTP Live Streaming](https://developer.apple.com/streaming/)
+* [Apple HLS Authoring Specification: General Authoring Requirements](https://developer.apple.com/library/content/documentation/General/Reference/HLSAuthoringSpec/Requirements.html)
+* [Apple HTTP Live Streaming Examples](https://developer.apple.com/streaming/examples/)
+* [RFC8216: HTTP Live Streaming](https://tools.ietf.org/html/rfc8216)

Vagrantfile

@@ -0,0 +1,17 @@
+Vagrant.configure(2) do |config|
+  config.vm.box = 'ubuntu-16.04-amd64'
+
+  config.vm.hostname = 'streaming'
+
+  config.vm.provider 'virtualbox' do |vb|
+    vb.linked_clone = true
+    vb.memory = 3072
+    vb.cpus = 2
+  end
+
+  config.vm.network "private_network", ip: "10.0.0.2"
+
+  config.vm.provision 'shell', path: 'provision-base.sh'
+  config.vm.provision 'shell', path: 'provision-nginx-rtmp-module.sh'
+  config.vm.provision 'shell', path: 'provision-videos.sh'
+end

config/etc/inputrc

@@ -0,0 +1,8 @@
+set input-meta on
+set output-meta on
+set show-all-if-ambiguous on
+set completion-ignore-case on
+"\e[A": history-search-backward
+"\e[B": history-search-forward
+"\eOD": backward-word
+"\eOC": forward-word

config/etc/profile.d/login.sh

@@ -0,0 +1,7 @@
+[[ "$-" != *i* ]] && return
+export EDITOR=vim
+export PAGER=less
+alias l='ls -lF --color'
+alias ll='l -a'
+alias h='history 25'
+alias j='jobs -l'

config/etc/vim/vimrc.local

@@ -0,0 +1,23 @@
+syntax on
+set background=dark
+set esckeys
+set ruler
+set laststatus=2
+set nobackup
+
+autocmd BufNewFile,BufRead Vagrantfile set ft=ruby
+autocmd BufNewFile,BufRead *.config set ft=xml
+
+" Usefull setting for working with Ruby files.
+autocmd FileType ruby set tabstop=2 shiftwidth=2 smarttab expandtab softtabstop=2 autoindent
+autocmd FileType ruby set smartindent cinwords=if,elsif,else,for,while,try,rescue,ensure,def,class,module
+
+" Usefull setting for working with Python files.
+autocmd FileType python set tabstop=4 shiftwidth=4 smarttab expandtab softtabstop=4 autoindent
+" Automatically indent a line that starts with the following words (after we press ENTER).
+autocmd FileType python set smartindent cinwords=if,elif,else,for,while,try,except,finally,def,class
+
+" Usefull setting for working with Go files.
+autocmd FileType go set tabstop=4 shiftwidth=4 smarttab expandtab softtabstop=4 autoindent
+" Automatically indent a line that starts with the following words (after we press ENTER).
+autocmd FileType go set smartindent cinwords=if,else,switch,for,func

provision-base.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# abort this script on errors.
+set -eux
+
+# prevent apt-get et al from opening stdin.
+# NB even with this, you'll still get some warnings that you can ignore:
+#     dpkg-preconfigure: unable to re-open stdin: No such file or directory
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install -y git-core
+apt-get install -y unzip xz-utils
+apt-get install -y --no-install-recommends httpie
+apt-get install -y --no-install-recommends vim
+apt-get install -y --no-install-recommends jq
+
+# set system configuration.
+rm -f /{root,home/*}/.{profile,bashrc}
+cp -v -r /vagrant/config/etc/* /etc
+
+su vagrant -c bash <<'VAGRANT_EOF'
+#!/bin/bash
+# abort this script on errors.
+set -eux
+
+# configure git.
+# see http://stackoverflow.com/a/12492094/477532
+git config --global user.name 'Rui Lopes'
+git config --global user.email 'rgl@ruilopes.com'
+git config --global push.default simple
+#git config --list --show-origin
+VAGRANT_EOF
+
+apt-get autoremove -y --purge

provision-nginx-rtmp-module.sh

@@ -0,0 +1,154 @@
+#!/bin/bash
+set -eux
+
+# add the nginx user.
+groupadd --system nginx-rtmp
+adduser \
+    --system \
+    --disabled-login \
+    --no-create-home \
+    --gecos '' \
+    --ingroup nginx-rtmp \
+    --home /opt/nginx-rtmp \
+    nginx-rtmp
+install -d -o root -g root -m 755 /opt/nginx-rtmp
+install -d -o root -g root -m 755 /opt/nginx-rtmp/public
+
+# download and install the latest version of ffmpeg.
+wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-64bit-static.tar.xz
+tar xf ffmpeg-release-64bit-static.tar.xz
+cp ffmpeg-*-static/{ffmpeg,ffprobe} /usr/local/bin
+ffmpeg -version
+
+# download the latest version of nginx-rtmp-module.
+git clone https://github.com/sergey-dryabzhinsky/nginx-rtmp-module.git
+
+# download, build and install nginx+nginx-rtmp-module.
+wget -q https://nginx.org/download/nginx-1.13.4.tar.gz
+tar xf nginx-1.13.4.tar.gz
+pushd nginx-1.13.4
+apt-get install -y libpcre3 libpcre3-dev libssl-dev
+./configure \
+    --prefix=/opt/nginx-rtmp \
+    --build=nginx-rtmp \
+    --user=nginx-rtmp \
+    --group=nginx-rtmp \
+    --add-module=../nginx-rtmp-module
+make -j 2
+make install #DESTDIR=$PWD/DIST
+popd
+
+# copy public data.
+cp -r /vagrant/public /opt/nginx-rtmp
+wget -qO /opt/nginx-rtmp/public/shaka-player.compiled.js https://cdnjs.cloudflare.com/ajax/libs/shaka-player/2.2.0/shaka-player.compiled.js
+wget -qO /opt/nginx-rtmp/public/hls.light.js https://github.com/video-dev/hls.js/raw/master/dist/hls.light.js
+wget -qO /opt/nginx-rtmp/public/hls.light.min.js https://github.com/video-dev/hls.js/raw/master/dist/hls.light.min.js
+wget -q https://github.com/videojs/video.js/releases/download/v6.2.7/video-js-6.2.7.zip
+unzip -d video-js-6.2.7 video-js-6.2.7.zip
+cp video-js-6.2.7/{video{,.min}.js,video-js{,.min}.css} /opt/nginx-rtmp/public
+wget -qO /opt/nginx-rtmp/public/videojs-contrib-hls.js https://github.com/videojs/videojs-contrib-hls/releases/download/v5.10.0/videojs-contrib-hls.js
+wget -qO /opt/nginx-rtmp/public/videojs-contrib-hls.min.js https://github.com/videojs/videojs-contrib-hls/releases/download/v5.10.0/videojs-contrib-hls.min.js
+cp nginx-rtmp-module/stat.xsl /opt/nginx-rtmp/public
+
+# create a tiny tmpfs for storing the video fragments.
+cat >>/etc/fstab <<EOF
+tmpfs /opt/nginx-rtmp/fragments tmpfs rw,nodev,nosuid,noexec,noatime,uid=0,gid=$(id -g nginx-rtmp),mode=1770,size=512M 0 0
+EOF
+mkdir /opt/nginx-rtmp/fragments
+mount /opt/nginx-rtmp/fragments
+
+# set the configuration.
+# see https://github.com/arut/nginx-rtmp-module/wiki/Directives
+cat >/opt/nginx-rtmp/conf/nginx.conf <<'EOF'
+#error_log stderr warn;
+
+worker_processes  auto;
+events {
+    worker_connections 1024;
+}
+
+rtmp {
+    server {
+        listen 1935;
+        application hls {
+            live on;
+            hls on;
+            hls_nested on;
+            hls_fragment 3s;
+            hls_playlist_length 3m;
+            hls_path /opt/nginx-rtmp/fragments/hls;
+            #allow publish 127.0.0.1;
+            #deny publish all;
+            #deny play all;
+        }
+    }
+}
+
+http {
+    sendfile on;
+    tcp_nopush on;
+    root /opt/nginx-rtmp/public;
+
+    types {
+        text/html html;
+        text/css css;
+        text/javascript js;
+        text/xsl xsl;
+        application/dash+xml mpd;
+        application/vnd.apple.mpegurl m3u8;
+        video/mp2t ts;
+    }
+    default_type application/octet-stream;
+
+    server {
+        listen 80;
+
+        location = /stat {
+            rtmp_stat all;
+            rtmp_stat_stylesheet stat.xsl;
+        }
+
+        location /hls/ {
+            #add_header Cache-Control no-cache;
+            root /opt/nginx-rtmp/fragments;
+        }
+    }
+}
+EOF
+/opt/nginx-rtmp/sbin/nginx -t
+
+# run as a service.
+cat >/etc/systemd/system/nginx-rtmp.service <<'EOF'
+[Unit]
+Description=nginx-rtmp
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=/opt/nginx-rtmp/sbin/nginx -g 'daemon off;'
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# start nginx.
+systemctl enable nginx-rtmp
+systemctl start nginx-rtmp
+
+# configure log rotation.
+cat >/etc/logrotate.d/nginx-rtmp <<'EOF'
+/opt/nginx-rtmp/logs/*.log {
+    daily
+    missingok
+    rotate 14
+    compress
+    delaycompress
+    notifempty
+    create 0640 root root
+    sharedscripts
+    postrotate
+        systemctl kill --signal=USR1 --kill-who=main nginx-rtmp
+    endscript
+}
+EOF

provision-videos.sh

@@ -0,0 +1,125 @@
+
+apt-get install -y fonts-dejavu-core
+
+wget -q -O /usr/local/bin/youtube-dl https://yt-dl.org/downloads/latest/youtube-dl
+chmod +x /usr/local/bin/youtube-dl
+
+wget -q -O /usr/local/bin/iframe-probe.py https://gist.githubusercontent.com/use-sparingly/7041ee993adb5c911f90/raw/d6cfe6a51c990ff5ae5242cb5711d2a68651f573/iframe-probe.py
+chmod +x /usr/local/bin/iframe-probe.py
+
+mkdir videos
+cd videos
+
+convert_video() {
+    input=$1; shift
+    output=$1; shift
+    max_video_height=$1; shift
+    fragment_length_seconds=$1; shift
+    fps=$1; shift
+    gop=$(expr $fragment_length_seconds \* $fps)   
+
+    echo "Transcoding $input into $output (${max_video_height}p)..."
+    # transcode (and scale) the original file to be directly consumed by the
+    # nginx-rtmp-module.
+    # NB the players are quite fragile, so its safer to use a constant GOP
+    #    length, no scene detection (bc it causes the keyframe interval to
+    #    vary) and closed GOPs.
+    # see https://trac.ffmpeg.org/wiki/Scaling%20(resizing)%20with%20ffmpeg
+    # see https://trac.ffmpeg.org/wiki/Encode/H.264
+    # see https://kvssoft.wordpress.com/2015/01/28/mpeg-dash-gop/
+    # see http://caniuse.com/#feat=mpeg4
+    extra_filter_v="drawtext=text='%{pts\\:hms} #%{n}':x=-5:y=3:fontsize=13:fontcolor=white:box=1:boxborderw=3:boxcolor=black:fontfile=/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf"
+    ffmpeg \
+        -loglevel info \
+        -y \
+        -i $input \
+        -r $fps \
+        -codec:v libx264 \
+        -preset medium \
+        -profile:v high -level 4.2 \
+        -crf 23 \
+        -g $gop \
+        -keyint_min $gop \
+        -sc_threshold 0 \
+        -flags +cgop \
+        -movflags +faststart \
+        -filter:v "scale=w=-2:h=$max_video_height,$extra_filter_v" \
+        -codec:a aac \
+        -q:a 4 \
+        -f flv \
+        -vstats_file ${output}_${max_video_height}p_stats.txt \
+        ${output}_${max_video_height}p.flv \
+        2>&1 \
+            | grep -vEe 'Past duration [0-9.]+ too large' \
+            | grep -vEe ' dropping frame '
+
+    echo "Dumping GOPs..."
+    # NB this calls ffprobe -show_frames -print_format json costa_rica_${max_video_height}p.flv
+    iframe-probe.py ${output}_${max_video_height}p.flv >${output}_${max_video_height}p_gops.txt
+    printf "   GOPs size\ttype\n"; awk '{print $3 "\t" $4}' ${output}_${max_video_height}p_gops.txt | sort | uniq -c
+
+    echo "Converting to static hls at /opt/nginx-rtmp/public/vod/hls/${output}/index.m3u8..."
+    rm -rf /opt/nginx-rtmp/public/vod/hls/${output}
+    mkdir -p /opt/nginx-rtmp/public/vod/hls/${output}
+    ffmpeg \
+        -loglevel info \
+        -i ${output}_${max_video_height}p.flv \
+        -codec:v copy \
+        -codec:a copy \
+        -hls_time $fragment_length_seconds \
+        -hls_list_size 0 \
+        -f hls \
+        /opt/nginx-rtmp/public/vod/hls/${output}/index.m3u8
+}
+
+echo 'Downloading the Costa Rica video....'
+youtube-dl -o costa_rica_720p.webm -f 'best[height=720]' 'https://www.youtube.com/watch?v=iNJdPyoqt8U'
+convert_video costa_rica_720p.webm costa_rica 240 3 24
+
+echo 'Downloading the Kung Fu Mantis vs Jumping Spider video....'
+youtube-dl -o kung_fu_mantis_vs_jumping_spider_720p.webm -f 'best[height=720]' 'https://www.youtube.com/watch?v=7wKu13wmHog'
+convert_video kung_fu_mantis_vs_jumping_spider_720p.webm kung_fu_mantis_vs_jumping_spider 240 3 24
+
+echo 'Downloading the Planet Earth II Continues video....'
+youtube-dl -o planet_earth_ii_continues_trailer_720p.webm -f 'best[height=720]' 'https://www.youtube.com/watch?v=h8yo_Sp-rGY'
+convert_video planet_earth_ii_continues_trailer_720p.webm planet_earth_ii_continues_trailer 240 3 24
+
+echo 'Downloading the Tears of Steel video....'
+wget -q http://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_720p.mov
+convert_video tears_of_steel_720p.mov tears_of_steel 240 3 24
+
+# continually stream the videos to nginx-rtmp in a background service.
+cat >stream-from-files.sh <<'EOF'
+#!/bin/bash
+set -eux
+while true; do
+    for n in *_240p.flv; do
+        ffmpeg \
+            -loglevel info \
+            -re \
+            -i $n \
+            -codec:v copy \
+            -codec:a copy \
+            -f flv \
+            rtmp://localhost:1935/hls/live
+        sleep 1
+    done
+done
+EOF
+chmod +x stream-from-files.sh
+cat >/etc/systemd/system/stream-from-files.service <<EOF
+[Unit]
+Description=stream-from-files
+After=network.target
+
+[Service]
+Type=simple
+WorkingDirectory=$PWD
+ExecStart=$PWD/stream-from-files.sh
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+EOF
+systemctl enable stream-from-files
+systemctl start stream-from-files