Run lettuce against prod release (#41)

lettuce: run tests against prod stack, options to control which tests are run

Dockerfile-build

@@ -22,12 +22,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   zlib1g-dev \
   && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-COPY docker-entrypoint.sh /app/docker-entrypoint.sh
+COPY docker-entrypoint-build.sh /app/docker-entrypoint-build.sh
 
 VOLUME ["/data"]
 
 ENV HOME /data
 WORKDIR /data
 
-ENTRYPOINT ["/app/docker-entrypoint.sh"]
+ENTRYPOINT ["/app/docker-entrypoint-build.sh"]
 CMD ["releasetarball"]

Jenkinsfile

@@ -23,7 +23,7 @@ node {
 
     stage 'Lettuce tests'
         wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) {
-            sh './develop.sh lettuce'
+            sh './develop.sh dev_lettuce'
         }
         step([$class: 'JUnitResultArchiver', testResults: '**/data/selenium/*.xml'])
         step([$class: 'ArtifactArchiver', artifacts: '**/data/selenium/*.png'])
@@ -35,6 +35,11 @@ node {
                 sh './develop.sh prod_build'
             }
 
+        stage 'Prod lettuce tests'
+            wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) {
+                sh './develop.sh prod_lettuce'
+            }
+
         stage 'Publish docker image'
             withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'dockerbot',
                               usernameVariable: 'DOCKER_USERNAME',

develop.sh

@@ -94,7 +94,7 @@ dev_build)
 releasetarball)
     create_release_tarball
     ;;
-start_prod)
+prod)
     start_prod
     ;;
 prod_build)
@@ -143,11 +143,14 @@ ci_docker_staging)
 docker_staging_lettuce)
     docker_staging_lettuce
     ;;
+dev_lettuce)
+    dev_lettuce
+    ;;
 lettuce)
-    create_base_image
-    create_build_image
-    create_dev_image
-    lettuce
+    dev_lettuce
+    ;;
+prod_lettuce)
+    prod_lettuce
     ;;
 *)
     usage

docker-compose-common.yml

@@ -92,7 +92,7 @@ runserver:
 
 uwsgi:
   image: muccg/rdrf-dev
-  command: uwsgi
+  command: uwsgi_local
   environment:
     - DBUSER=rdrfapp
     - WAIT_FOR_DB=1

docker-compose-lettuce.yml

@@ -5,19 +5,40 @@ dataselenium:
     service: dataselenium
 
 # runs the lettuce tests against docker-compose-teststack.yml
-lettucehost:
+devlettuce:
   image: muccg/rdrf-dev
   command: lettuce
   environment:
     - DBUSER=rdrfapp
     - WAIT_FOR_RUNSERVER=1
     - WAIT_FOR_DB=1
     - WAIT_FOR_MONGO=1
+    - DEBUG=0
   volumes_from:
-    - dataselenium      
+    - dataselenium
   external_links:
     - rdrf_runservertest_1:web
     - rdrf_seleniumhub_1:hub
     - rdrf_dbtest_1:db
     - rdrf_mongotest_1:mongo
 
+# runs the lettuce tests against docker-compose-teststack.yml
+prodlettuce:
+  image: muccg/rdrf-dev
+  command: lettuce
+  environment:
+    - DBUSER=rdrfapp
+    - WAIT_FOR_UWSGI=1
+    - WAIT_FOR_DB=1
+    - WAIT_FOR_MONGO=1
+    - TEST_APP_PORT=8443
+    - TEST_APP_SCHEME=https
+    - TEST_APP_PATH=/app/
+    - DEBUG=0
+  volumes_from:
+    - dataselenium
+  external_links:
+    - rdrf_uwsgiprod_1:uwsgi
+    - rdrf_seleniumhub_1:hub
+    - rdrf_dbprod_1:db
+    - rdrf_mongoprod_1:mongo

docker-compose-prod.yml

@@ -40,16 +40,20 @@ nginxprod:
 
 uwsgiprod:
   image: muccg/rdrf:${GIT_TAG}
-  command: uwsgi_fixtures
+  command: uwsgi_local
   environment:
     - DBUSER=rdrfapp
     - WAIT_FOR_DB=1
     - WAIT_FOR_CACHE=1
     - WAIT_FOR_MONGO=1
+    - DEPLOYMENT=test
+    - ALLOWED_HOSTS=*
   volumes_from:
     - dataprod
   links:
     - dbprod:db
     - cacheprod:cache
     - reportingprod:reporting
     - mongoprod:mongo
+  ports:
+    - "8433:8433"

docker-compose-teststack.yml

@@ -31,7 +31,9 @@ runservertest:
     service: runserver
   volumes_from:
     - datatest
-  command: runserverlettuce
+  command: runserver
+  environment:
+    - DEPLOYMENT=test
   links:
     - dbtest:db
     - reportingtest:reporting

docker-entrypoint-build.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+trap exit SIGHUP SIGINT SIGTERM
+env | grep -iv PASS | sort
+
+# prepare a tarball of build
+if [ "$1" = 'releasetarball' ]; then
+    echo "[Run] Preparing a release tarball"
+
+    set -e
+    cd /app
+    rm -rf /app/*
+    echo $GIT_TAG
+    set -x
+    git clone --depth=1 --branch=${GIT_TAG} ${PROJECT_SOURCE} .
+    git ls-remote ${PROJECT_SOURCE} ${GIT_TAG} > .version
+
+    # Note: Environment vars are used to control the behaviour of pip (use local devpi for instance)
+    pip install -e rdrf
+    set +x
+
+    # create release tarball
+    DEPS="/env /app/uwsgi /app/docker-entrypoint.sh /app/rdrf /app/scripts"
+    cd /data
+    exec tar -cpzf ${PROJECT_NAME}-${GIT_TAG}.tar.gz ${DEPS}
+fi
+
+echo "[RUN]: Builtin command not provided [releasetarball]"
+echo "[RUN]: $@"
+
+exec "$@"

docker-entrypoint.sh

@@ -32,14 +32,15 @@ function wait_for_services {
     if [[ "$WAIT_FOR_MONGO" ]] ; then
         dockerwait $MONGOSERVER $MONGOPORT
     fi
-
     if [[ "$WAIT_FOR_REPORTING" ]]; then
         dockerwait $REPORTINGDBSERVER $REPORTINGDBPORT
     fi
-
     if [[ "$WAIT_FOR_HOST_PORT" ]]; then
         dockerwait $DOCKER_ROUTE $WAIT_FOR_HOST_PORT
     fi
+    if [[ "$WAIT_FOR_UWSGI" ]] ; then
+        dockerwait $UWSGISERVER $UWSGIPORT
+    fi
 }
 
 
@@ -58,35 +59,29 @@ function defaults {
     : ${REPORTINGDBNAME:=${REPORTINGDBUSER}}
     : ${REPORTINGDBPASS:=${REPORTINGDBUSER}}
 
+    : ${UWSGISERVER:="uwsgi"}
+    : ${UWSGIPORT:="9000"}
     : ${RUNSERVER:="web"}
     : ${RUNSERVERPORT:="8000"}
-    : ${SELENIUMRUNSERVERPORT:="18000"}
     : ${CACHESERVER:="cache"}
     : ${CACHEPORT:="11211"}
     : ${MEMCACHE:="${CACHESERVER}:${CACHEPORT}"}
     : ${MONGOSERVER:="mongo"}
     : ${MONGOPORT:="27017"}
 
-    export DBSERVER DBPORT DBUSER DBNAME DBPASS MONGOSERVER MONGOPORT MEMCACHE DOCKER_ROUTE
-    export REPORTINGDBSERVER REPORTINGDBPORT REPORTINGDBUSER REPORTINGDBNAME REPORTINGDBPASS
-}
+    # variables to control where tests will look for the app (lettuce via selenium hub)
+    : ${TEST_APP_SCHEME:="http"}
+    : ${TEST_APP_HOST:=${DOCKER_ROUTE}}
+    : ${TEST_APP_PORT:="18000"}
+    : ${TEST_APP_PATH:="/"}
+    : ${TEST_APP_URL:="${TEST_APP_SCHEME}://${TEST_APP_HOST}:${TEST_APP_PORT}${TEST_APP_PATH}"}
 
+    #: ${TEST_BROWSER:="chrome"}
+    : ${TEST_BROWSER:="firefox"}
 
-function selenium_defaults {
-    : ${RDRF_URL:="http://$DOCKER_ROUTE:$SELENIUMRUNSERVERPORT/"}
-    #: ${RDRF_BROWSER:="*googlechrome"}
-    : ${RDRF_BROWSER:="*firefox"}
-
-    if [ ${DEPLOYMENT} = "prod" ]; then
-        RDRF_URL="https://$DOCKER_ROUTE:8443/app/"
-    fi
-
-    # stellar config needs to be in PWD at runtime for lettuce tests
-    if [ ! -f ${PWD}/stellar.yaml ]; then
-        cp /app/stellar.yaml ${PWD}/stellar.yaml
-    fi
-
-    export RDRF_URL RDRF_BROWSER
+    export DBSERVER DBPORT DBUSER DBNAME DBPASS MONGOSERVER MONGOPORT MEMCACHE DOCKER_ROUTE
+    export REPORTINGDBSERVER REPORTINGDBPORT REPORTINGDBUSER REPORTINGDBNAME REPORTINGDBPASS
+    export TEST_APP_URL TEST_APP_SCHEME TEST_APP_HOST TEST_APP_PORT TEST_APP_PATH TEST_BROWSER
 }
 
 
@@ -125,37 +120,25 @@ function _rdrf_import_grdr {
     django-admin.py import_registry --file=/app/grdr.yaml
 }
 
+function _django_fixtures {
+    if [ "${DEPLOYMENT}" = 'test' ]; then
+        _django_iprestrict_permissive_fixtures
+    fi
+
+    if [ "${DEPLOYMENT}" = 'dev' ]; then
+        _django_dev_fixtures
+    fi
+}
+
 
 trap exit SIGHUP SIGINT SIGTERM
 defaults
 env | grep -iv PASS | sort
 wait_for_services
 
-# prepare a tarball of build
-if [ "$1" = 'releasetarball' ]; then
-    echo "[Run] Preparing a release tarball"
-
-    set -e
-    cd /app
-    rm -rf /app/*
-    echo $GIT_TAG
-    set -x
-    git clone --depth=1 --branch=${GIT_TAG} ${PROJECT_SOURCE} .
-    git ls-remote ${PROJECT_SOURCE} ${GIT_TAG} > .version
-
-    # Note: Environment vars are used to control the behaviour of pip (use local devpi for instance)
-    pip install -e rdrf
-    set +x
-
-    # create release tarball
-    DEPS="/env /app/uwsgi /app/docker-entrypoint.sh /app/rdrf /app/scripts"
-    cd /data
-    exec tar -cpzf ${PROJECT_NAME}-${GIT_TAG}.tar.gz ${DEPS}
-fi
-
-# uwsgi entrypoint
+# prod uwsgi entrypoint
 if [ "$1" = 'uwsgi' ]; then
-    echo "[Run] Starting uwsgi"
+    echo "[Run] Starting prod uwsgi"
 
     : ${UWSGI_OPTS="/app/uwsgi/docker.ini"}
     echo "UWSGI_OPTS is ${UWSGI_OPTS}"
@@ -167,16 +150,16 @@ if [ "$1" = 'uwsgi' ]; then
     exec uwsgi --die-on-term --ini ${UWSGI_OPTS}
 fi
 
-# uwsgi entrypoint, with fixtures, intended for use in local environment only
-if [ "$1" = 'uwsgi_fixtures' ]; then
-    echo "[Run] Starting uwsgi with fixtures"
+# local and test uwsgi entrypoint
+if [ "$1" = 'uwsgi_local' ]; then
+    echo "[Run] Starting local uwsgi"
 
     : ${UWSGI_OPTS="/app/uwsgi/docker.ini"}
     echo "UWSGI_OPTS is ${UWSGI_OPTS}"
 
     _django_collectstatic
     _django_migrate
-    _django_dev_fixtures
+    _django_fixtures
     _django_check_deploy
 
     exec uwsgi --die-on-term --ini ${UWSGI_OPTS}
@@ -191,7 +174,7 @@ if [ "$1" = 'runserver' ]; then
 
     _django_collectstatic
     _django_migrate
-    _django_dev_fixtures
+    _django_fixtures
 
     echo "running runserver ..."
     exec django-admin.py ${RUNSERVER_OPTS}
@@ -206,30 +189,13 @@ if [ "$1" = 'grdr' ]; then
 
     _django_collectstatic
     _django_migrate
-    _django_dev_fixtures
+    _django_fixtures
     _rdrf_import_grdr
 
     echo "running runserver ..."
     exec django-admin.py ${RUNSERVER_OPTS}
 fi
 
-# runserver lettuce entrypoint
-if [ "$1" = 'runserverlettuce' ]; then
-    echo "[Run] Starting runserver"
-
-    : ${RUNSERVER_OPTS="runserver_plus 0.0.0.0:${RUNSERVERPORT} --settings=${DJANGO_SETTINGS_MODULE}"}
-    echo "RUNSERVER_OPTS is ${RUNSERVER_OPTS}"
-
-    _django_collectstatic
-    _django_migrate
-    _django_iprestrict_permissive_fixtures
-
-    echo "running runserver ..."
-    exec django-admin.py ${RUNSERVER_OPTS}
-fi
-
-
-
 # runtests entrypoint
 if [ "$1" = 'runtests' ]; then
     echo "[Run] Starting tests"
@@ -239,9 +205,14 @@ fi
 # lettuce entrypoint
 if [ "$1" = 'lettuce' ]; then
     echo "[Run] Starting lettuce"
-    selenium_defaults
+
+    # stellar config needs to be in PWD at runtime for lettuce tests
+    if [ ! -f ${PWD}/stellar.yaml ]; then
+        cp /app/stellar.yaml ${PWD}/stellar.yaml
+    fi
     rm -f /data/*.png
-    exec django-admin.py run_lettuce --with-xunit --xunit-file=/data/tests.xml
+    shift
+    exec django-admin.py run_lettuce --with-xunit --xunit-file=/data/tests.xml $@
 fi
 
 echo "[RUN]: Builtin command not provided [tarball|lettuce|runtests|runserver|uwsgi|uwsgi_fixtures]"