Minimum CI build integration

.pylintrc

@@ -0,0 +1,5 @@
+[MASTER]
+good-names=fd,e
+output-format=parseable
+disable=F0401,R0201,R0902,W0703,I0011,W0403,E1102,R0903,E0611,W0511
+max-line-length=100

.travis.yml

@@ -0,0 +1,33 @@
+sudo: required
+dist: trusty
+language: python
+python:
+  - "2.7"
+services:
+  - docker
+addons:
+  apt:
+    update: true
+    packages:
+      - openvswitch-switch
+install:
+  - set -e
+  - bin/setup_base
+  - bin/setup_dev
+script:
+  - set -e
+  - bin/build_daq
+  - bin/test_daq
+env:
+  global:
+    - DOCKER_STARTUP_TIMEOUT_MS=60000
+  matrix:
+    - DAQ_CODECOV=y
+branches:
+  except:
+  - /^release-\d+\.\d+(\.\d+)?$/
+deploy:
+  provider: script
+  script: bash bin/docker_push
+  on:
+    branch: release

bin/setup_dev

@@ -31,7 +31,7 @@ source venv/bin/activate
 
 # Python extension packages.
 pip install --index-url=https://pypi.python.org/simple \
-    netifaces google-cloud-pubsub==0.32.1
+    netifaces codecov google-cloud-pubsub==0.32.1
 
 # Hack to make pylint happy.
 touch venv/lib/python2.7/site-packages/google/__init__.py

cmd/build

@@ -0,0 +1,103 @@
+#!/bin/bash -e
+
+# Set this so docker build will fail even though piped through tee.
+set -o pipefail
+
+ROOT=$(dirname $0)/..
+cd $ROOT
+
+local_config=local/system.conf
+build_root=build
+in_container=$DAQ_CONTAINER
+build_args=
+do_images=
+ext_msg=
+inline=
+copydirs="bin cmd misc faucet docker daq"
+
+if [ "$1" == save ]; then
+    do_images=save
+    ext_msg=/save
+    shift
+elif [ "$1" == load ]; then
+    do_images=load
+    shift
+fi
+
+if [ "$1" == inline ]; then
+    inline=y
+    shift
+fi
+
+if [ -f $local_config ]; then
+  echo Loading build configuraiton from $local_config
+  source $local_config
+fi
+
+if [ -n "$in_container" ]; then
+    # Run in container for pylint versioning.
+    echo Running style checks...
+    bin/check_style
+
+    echo 'DOCKER_OPTS="-s vfs -D -l debug"' > /etc/default/docker
+    service docker start
+    sleep 3
+    echo Importing current local version of daq...
+    ROOT=/root/daq
+    rm -rf $ROOT
+    mkdir -p $ROOT
+    cd /daq
+    for subdir in $copydirs; do
+        echo Copying subdir $subdir...
+        cp -a $subdir $ROOT/
+    done
+    cd $ROOT
+    if [ "$do_images" != load ]; then
+        rm -rf build
+    fi
+    echo Now running in `pwd`
+fi
+
+mkdir -p local
+date > local/last_build.txt
+sudo rm -rf $build_root
+mkdir -p $build_root
+
+for file in docker/Dockerfile.*; do
+    if [ ${file%\~} != ${file} ]; then
+        continue
+    fi
+    module=${file#docker/Dockerfile.}
+    if [ "$module" == base -a -n "$in_container" ]; then
+        continue
+    fi
+    logfile=$build_root/docker_build.$module
+    failed=
+    tag=daq/$module
+    if [ "$do_images" == load ]; then
+        docker load -i /daq/$build_root/docker_image.$module
+    else
+        echo Build$ext_msg $file into daq/$module, log to $logfile...
+        if [ -n "$inline" ]; then
+            docker build -t $tag $build_args -f $file . 2>&1 | tee $logfile || failed=y
+        else
+            docker build -t $tag $build_args -f $file . > $logfile 2>&1 || failed=y
+        fi
+        if [ -n "$failed" ]; then
+            tail $logfile
+            echo Build failed, see $logfile for complete log.
+            false
+        fi
+        echo Build complete. >> $logfile
+        if [ "$do_images" == save ]; then
+            docker save $tag > $build_root/docker_image.$module
+        fi
+    fi
+done
+
+if [ -n "$in_container" ]; then
+    service docker stop
+    sleep 2
+    rm -fr /var/lib/docker/runtimes
+    rm -fr faucet/.git
+fi

cmd/clean

@@ -0,0 +1,15 @@
+#!/bin/bash -e
+
+ROOT=$(realpath $(dirname $0)/..)
+
+test -d $ROOT/daq && sudo rm -rf $ROOT/inst $ROOT/build
+
+docker ps -aq --no-trunc | xargs -r docker rm || true
+
+docker images | egrep ^daqf/ | awk '{print $1":"$2}' | xargs docker rmi || true
+
+docker images | egrep ^daq/ | awk '{print $1":"$2}' | xargs docker rmi || true
+
+docker images -q --filter dangling=true | xargs -r docker rmi || true
+
+docker system prune -f

cmd/exrun

@@ -0,0 +1,115 @@
+#!/bin/bash -e
+
+ROOT=$(realpath $(dirname $0)/..)
+INSTDIR=$ROOT/inst
+LOCALDIR=$ROOT/local
+FAUCET_SOCK=faucet_event.sock
+EVENT_SOCK=$INSTDIR/$FAUCET_SOCK
+SYSTEM_CONF=system.conf
+LOCAL_CONFIG=$LOCALDIR/$SYSTEM_CONF
+DAQ_VERSION=$(cat $ROOT/misc/RELEASE_VERSION)
+
+cd $ROOT
+
+FAUCET=$(realpath $ROOT/faucet)
+MININET=$(realpath $ROOT/mininet)
+
+if [ ! -f $LOCAL_CONFIG ]; then
+    mkdir -p $LOCALDIR
+    echo Copying default $SYSTEM_CONF to $LOCAL_CONFIG
+    cp misc/$SYSTEM_CONF $LOCAL_CONFIG
+fi
+
+echo Loading daq run configuration from $LOCAL_CONFIG
+source $LOCAL_CONFIG
+
+PYTHONPATH=$FAUCET:$MININET
+
+set | egrep ^[a-z][a-z_]*=
+
+if [ ! -d $FAUCET ]; then
+    echo Faucet directory $FAUCET not found.
+    false
+fi
+
+mkdir -p $INSTDIR
+rm -f $EVENT_SOCK
+
+docker ps > /dev/null 2>&1 || service docker start
+
+ovsctl=/usr/share/openvswitch/scripts/ovs-ctl
+$ovsctl status || sudo $ovsctl start
+
+if [ -z "$daq_intf" ]; then
+    echo Setting daq_intf not defined, defaulting to auto-start faux device.
+    daq_intf="faux!"
+fi
+
+intfs=$(echo $daq_intf | tr , ' ')
+for intf in $intfs; do
+    if [ "$intf" == "local!" ]; then
+        intf=local
+        echo Implicitly running local device...
+        $ROOT/cmd/local
+        cleanup="$ROOT/cmd/local clean"
+    fi
+
+    if [ "$intf" == "faux!" ]; then
+        intf=faux
+        echo Implicitly running faux device...
+        $ROOT/cmd/faux
+        cleanup="echo Killing daq-faux container...; docker kill daq-faux > /dev/null"
+    fi
+
+    ip -br addr show $intf || true
+
+    # If running inside of a container, wait until the host test interface shows up...
+    if [ -n "$DAQ_CONTAINER" -a -z "$cleanup" ]; then
+        retry=5
+        while ! ifconfig -a | fgrep -q $intf; do
+            if [ "$retry" == 0 ]; then
+                echo Failed to find interface $intf.
+                false
+            fi
+            echo Waiting for interface $intf to appear retry $retry...
+            sleep 1
+            retry=$((retry-1))
+        done
+    fi
+done
+
+if [ -f .pdbrc ]; then
+    echo Found .pdbrc file, using pdb...
+    runcmd=pdb
+elif [ -n "$DAQ_CODECOV" ]; then
+    echo Running with coverage analysis...
+    runcmd="coverage run"
+else
+    runcmd=python
+fi
+
+echo Executing sudo $runcmd "$@"...
+sudo -s -- <<EOF
+    set -e
+    if [ -d venv ]; then
+        echo Entering virtual python environment...
+        source venv/bin/activate
+    fi
+    env \
+        DAQ_CONTAINER=$DAQ_CONTAINER \
+        FAUCET_EVENT_SOCK=$EVENT_SOCK \
+        DAQ_VERSION=$DAQ_VERSION \
+        PYTHONDONTWRITEBYTECODE=True \
+        PYTHONPATH=$PYTHONPATH \
+        DOCKER_STARTUP_TIMEOUT_MS=$DOCKER_STARTUP_TIMEOUT_MS \
+        TERM=dumb $runcmd daq/daq.py $LOCAL_CONFIG "$@"
+    if [ -n "$DAQ_CODECOV" ]; then
+        codecov
+    fi
+EOF
+
+if [ -n "$cleanup" ]; then
+    sh -c "$cleanup"
+fi
+
+echo Done with run.

cmd/faux

@@ -0,0 +1,68 @@
+#!/bin/bash -e
+
+ROOT=$(realpath $(dirname $0)/..)
+INSTDIR=$ROOT/inst
+IMAGE=daq/faux
+local_config=$ROOT/local/system.conf
+postfix=
+
+# If the first argument is a number, then it's a port designation.
+if [ $# -gt 0 -a $(($1)) -gt 0 ]; then
+    postfix=-$1
+    shift
+fi
+
+
+if ifconfig "$1" > /dev/null 2>&1 ; then
+    realintf=$1
+    postfix=-$realintf
+    echo Running using interface $realintf
+    shift
+fi
+
+FAUX_ARGS="$*"
+
+if [ -f $local_config ]; then
+  echo Loading faux configuraiton from $local_config
+  source $local_config
+fi
+
+CONTAINER=daq-faux$postfix
+
+if [ -z "$FAUX_ARGS" ]; then
+    FAUX_ARGS="$faux_args"
+fi
+
+echo Launching faux $FAUX_ARGS...
+
+mkdir -p $INSTDIR
+
+if docker container inspect $CONTAINER --format '{{ .Name }}' > /dev/null 2>&1; then
+    echo -n "Clensing old container "
+    docker rm -f $CONTAINER
+fi
+
+docker_args="\
+        -v $INSTDIR/faux:/tmp \
+        "
+
+cid=$(docker run -d --privileged --name $CONTAINER --net=none $docker_args $IMAGE $FAUX_ARGS)
+pid=$(docker inspect --format="{{ .State.Pid }}" $cid)
+
+if [ -n "$realintf" ]; then
+    echo Moving $realintf into $pid...
+    sudo ip link set $realintf netns $pid
+    docker exec $CONTAINER ip link set $realintf name faux-eth0
+    docker exec $CONTAINER ip link set faux-eth0 up
+    docker exec $CONTAINER ifconfig faux-eth0
+else
+    intf=faux$postfix
+    echo Removing old interface $intf
+    sudo ip link del $intf 2>/dev/null || true
+
+    echo Adding new interface to $pid...
+    sudo ip link add $intf type veth peer name faux-eth0 netns $pid
+    sudo ip link set $intf up
+fi
+
+echo Done with faux device launch.