new script for convenient local testing

.travis.yml

@@ -12,6 +12,7 @@ env:
   - secure: WS9nwlZfrSgdU5bUw9ZXh5xD067PyUVq+UZnPazORgh5kgkTTX4EsNnXPtmUiQ9DOwVvBiCZzn6c7N+YPuUdEROOR/pBjtFA+w/APFk4ug1tcIzRH7z2we+fZaAnKK3SRqOhHkUYa8tjZFGr53RsaKaxKEc0SIqsHGbDupdBz9KC4raXyUQobcLxWbE6BQq2jhyJGqWxOrhTTrd3WCRKVTgjUlQmwimk6s9txfnEbEguE7UlRhlO0CSqgBj2oUvwtJQSkZWAFminUi2Jcmnduebu42iYX6IF3bYNLVxy8d56v2APGv6reipsD9m3l6URRtpgGygMF4EI9X6dlCjT2qMrwwuWQcZJncFkR584dL+MiPfA2RXamGQybX8HSlnrn60ZKzr5gmw4hetffUGwmOA2VlEW1Wcxcwcef+/8osWwnfLT4I3i736P71LARcBWjBaXKKCXCkCLegv0WB0V/JXyo6IfdLToB0ElAwNfoSlhK1ZtTejngMQewzh6HFcyJwbyPog8/WCL+/j+OohdLZYgiXuldeoLoTwM8YmrSV7twUg3rl2Uc9AWd6iKXC2n8t+2XXdSTwcyrLAEp5J2gZA+HPE/4R6/K1YLbILfMp+u2k8TpazSVX7oRwIcp5UJK6NGx9KDLnT5nJXFW2J5SuBdvQjOXGNSVxyAhBg2AAM=
   - SUBDAGS=2
   - BIOCONDA_UTILS_TAG=2f8fa70
+  - BIOCONDA_UTILS_ARGS=--loglevel=info
   matrix:
   - SUBDAG=0
   - SUBDAG=1

README.md

@@ -74,27 +74,15 @@ or by adding the channels to the build command:
 
     conda build recipes/your_package --channel bioconda --channel r --channel conda-forge
 
-Then, you can test it in the docker container. The authoritative source for how
-packages are built can be found in the `scripts/travis-run.sh` script, the
-`config.yml` config file, and the `.travis.yml` config file. The following
-script sets some environmental variables to simulate the travis-ci build and
-then calls the `travis-run.sh` script. It should be called from the top level
-of the repository.
+Then, you can test the build in a docker container. The authoritative source
+for how packages are built can be found in the `scripts/travis-run.sh` script,
+the `config.yml` config file, and the `.travis.yml` config file. However, the
+`simulate-travis.py` script can be used for conveniently testing on a local
+machine. For example,
 
-```bash
-    #!/bin/bash
-
-    export SUBDAGS=1
-    export TRAVIS_OS_NAME=linux
-    export TRAVIS_BRANCH=0
-    export TRAVIS_PULL_REQUEST=false
-    export SUBDAG=0
-
-    # set this to match .travis.yml
-    export BIOCONDA_UTILS_TAG=2f8fa70
-    scripts/travis-run.sh
 ```
-
+./simulate-travis.py --packages mypackagename --loglevel=debug
+```
 
 If rebuilding a previously-built package and the version number hasn't changed,
 be sure to increment the build number in `meta.yaml` (the default build number

scripts/travis-run.sh

@@ -1,12 +1,11 @@
 #!/bin/bash
 set -euo pipefail
 export PATH=/anaconda/bin:$PATH
-
 if [[ $TRAVIS_OS_NAME = "linux" ]]
 then
-    docker run -e SUBDAG -e SUBDAGS -e TRAVIS_BRANCH -e TRAVIS_PULL_REQUEST -e ANACONDA_TOKEN \
+    docker run -e SUBDAG -e SUBDAGS -e TRAVIS_BRANCH -e TRAVIS_PULL_REQUEST -e ANACONDA_TOKEN -e BIOCONDA_UTILS_ARGS \
                -v `pwd`:/bioconda-recipes condaforge/linux-anvil /bin/bash -c \
-               "cd /bioconda-recipes; conda install --file scripts/requirements.txt; pip install git+https://github.com/bioconda/bioconda-utils.git@$BIOCONDA_UTILS_TAG; pip install git+https://github.com/bioconda/conda-build.git@fix-build-skip-existing; bioconda-utils build recipes config.yml --loglevel=info"
+               "cd /bioconda-recipes; conda install --file scripts/requirements.txt; pip install git+https://github.com/bioconda/bioconda-utils.git@$BIOCONDA_UTILS_TAG; pip install git+https://github.com/bioconda/conda-build.git@fix-build-skip-existing; set -x; bioconda-utils build recipes config.yml $BIOCONDA_UTILS_ARGS; set +x"
 
     if [[ $SUBDAG = 0 ]]
     then
@@ -20,5 +19,5 @@ else
     # build packages
     #scripts/build-packages.py --repository . --env-matrix scripts/env_matrix.yml
     pip install git+https://github.com/bioconda/conda-build.git@fix-build-skip-existing
-    bioconda-utils build recipes config.yml --loglevel=info
+    bioconda-utils build recipes config.yml $BIOCONDA_UTILS_ARGS
 fi

simulate-travis.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+import os
+import platform
+import sys
+import yaml
+import subprocess as sp
+import argparse
+
+usage = """
+
+This script simulates a travis-ci run on the local machine by using the current
+values in .travis.yml. It is intended to be run in the top-level directory of
+the bioconda-recipes repository.
+
+Any additional arguments to this script are interpreted as arguments to be
+passed to `bioconda-utils build`. For example, to build a single recipe (or
+glob of recipes):
+
+    simulate-travis.py --packages mypackagename bioconductor-*
+
+or modify the log level:
+
+    simulate-travis.py --packages mypackagename --loglevel=debug
+
+"""
+
+ap = argparse.ArgumentParser(usage=usage)
+args, extra = ap.parse_known_args()
+
+# Read in the current .travis.yml to ensure we're getting the right vars.
+# Mostly we care about the bioconda-utils git tag.
+travis_config = yaml.load(open('.travis.yml'))
+env = {}
+for var in travis_config['env']['global']:
+    if isinstance(var, dict) and list(var.keys()) == ['secure']:
+        continue
+    name, value = var.split('=', 1)
+    env[name] = value
+
+# SUBDAG is set by travis-ci according to the matrix in .travis.yml, so here we
+# force it to just use one. The default is to run two parallel jobs, but here
+# we set SUBDAGS to 1 so we only run a single job.
+#
+# See https://docs.travis-ci.com/user/speeding-up-the-build for more.
+env['SUBDAGS'] = '1'
+env['SUBDAG'] = '0'
+
+# These are set by the travis-ci environment; here we only set the variables
+# used by bioconda-utils.
+#
+# See https://docs.travis-ci.com/user/environment-variables for more.
+if platform.system() == 'Darwin':
+    env['TRAVIS_OS_NAME'] = 'osx'
+else:
+    env['TRAVIS_OS_NAME'] = 'linux'
+
+env['TRAVIS_BRANCH'] = 'false'
+env['TRAVIS_PULL_REQUEST'] = 'false'
+
+# Any additional arguments from the command line are added here.
+env['BIOCONDA_UTILS_ARGS'] += ' ' + ' '.join(extra)
+
+sp.check_call(['scripts/travis-run.sh'], env=env)