Merge branch 'master' of https://github.com/nipy/nipype into enh/crashtxt

Author: mgxd

.circle/codecov.sh

@@ -13,27 +13,27 @@ if [ "${CIRCLE_NODE_TOTAL:-}" != "4" ]; then
   exit 1
 fi
 
-# These tests are manually balanced based on previous build timings. 
+# These tests are manually balanced based on previous build timings.
 # They may need to be rebalanced in the future.
 case ${CIRCLE_NODE_INDEX} in
   0)
-    docker run --rm -it -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /src/nipype nipype/nipype:py27 /usr/bin/run_pytests.sh py27 && \
-    docker run --rm -it -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /src/nipype nipype/nipype:py35 /usr/bin/run_pytests.sh py35 && \
-    docker run --rm -it -v $WORKDIR:/work -w /src/nipype/doc nipype/nipype:py35 /usr/bin/run_builddocs.sh && \
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d && \
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_pytests.sh && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_pytests.sh && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $WORKDIR:/work -w /src/nipype/doc nipype/nipype:py35 /usr/bin/run_builddocs.sh && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d
     ;;
   1)
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 && \
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ level1 && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_dartel Linear /data/examples/ l2pipeline
     ;;
   2)
-    docker run --rm -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
-    docker run --rm -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
     ;;
   3)
-    docker run --rm -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline && \
-    docker run --rm -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_fsl_feeds Linear /data/examples/ l1pipeline && \
+    docker run --rm=false -it -e CODECOV_TOKEN=${CODECOV_TOKEN} -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py35 /usr/bin/run_examples.sh fmri_fsl_reuse Linear /data/examples/ level1_workflow
     ;;
 esac

.circle/tests.sh

@@ -1,7 +1,2 @@
 [run]
 branch = True
-include = */nipype/*
-omit =
-    */nipype/external/*
-    */nipype/fixes/*
-    */setup.py

.coveragerc

@@ -1,6 +1,7 @@
 # python cache
-__pycache__/**/*
 __pycache__
+**/__pycache__
+**/*.pyc
 *.pyc
 
 # python distribution
@@ -23,14 +24,14 @@ src/
 # other
 docs/**/*
 docs/
+.cache/
 .circle/**/*
 .circle/
 circle.yml
-.coverage
-.coveragerc
-codecov.yml
 rtd_requirements.txt
 Vagrantfile
 .travis.yml
-.noserc
+.mailmap
 
+# Previous coverage results
+.coverage

.dockerignore

@@ -42,7 +42,7 @@ before_install:
 install:
 - travis_retry pip install -e .[$NIPYPE_EXTRAS]
 script:
-- py.test --doctest-modules nipype
+- py.test -v --doctest-modules nipype
 deploy:
   provider: pypi
   user: satra

.noserc

@@ -71,6 +71,20 @@ RUN conda config --add channels conda-forge; sync && \
 RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/site-packages/matplotlib/mpl-data/matplotlibrc && \
     python -c "from matplotlib import font_manager"
 
+# Install codecov inside container
+RUN curl -so /usr/bin/codecov.io https://codecov.io/bash && \
+    chmod 755 /usr/bin/codecov.io
+
+# Install CI scripts
+COPY docker/files/run_* /usr/bin/
+RUN chmod +x /usr/bin/run_*
+
+# Replace imglob with a Python3 compatible version
+COPY nipype/external/fsl_imglob.py /usr/bin/fsl_imglob.py
+RUN rm -rf ${FSLDIR}/bin/imglob && \
+    chmod +x /usr/bin/fsl_imglob.py && \
+    ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob
+
 # Installing dev requirements (packages that are not in pypi)
 WORKDIR /src/
 COPY requirements.txt requirements.txt
@@ -83,16 +97,6 @@ RUN cd /src/nipype && \
     pip install -e .[all] && \
     rm -rf ~/.cache/pip
 
-# Install CI scripts
-COPY docker/files/run_* /usr/bin/
-RUN chmod +x /usr/bin/run_*
-
-# Replace imglob with a Python3 compatible version
-COPY nipype/external/fsl_imglob.py /usr/bin/fsl_imglob.py
-RUN rm -rf ${FSLDIR}/bin/imglob && \
-    chmod +x /usr/bin/fsl_imglob.py && \
-    ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob
-
 WORKDIR /work/
 
 ARG BUILD_DATE

.travis.yml

@@ -24,7 +24,7 @@ dependencies:
     - sudo apt-get -y update && sudo apt-get install -y wget bzip2
     # Create work folder and force group permissions
     - mkdir -p $WORKDIR && sudo setfacl -d -m group:ubuntu:rwx $WORKDIR && sudo setfacl -m group:ubuntu:rwx $WORKDIR
-    - mkdir -p $HOME/docker $HOME/examples $WORKDIR/pytest $WORKDIR/logs
+    - mkdir -p $HOME/docker $HOME/examples $WORKDIR/tests $WORKDIR/crashfiles $WORKDIR/logs ${CIRCLE_TEST_REPORTS}/tests/
   override:
     - if [[ -e $HOME/docker/cache.tar ]]; then docker load --input $HOME/docker/cache.tar; fi :
         timeout: 6000
@@ -49,13 +49,15 @@ test:
         timeout: 7200
         parallel: true
   post:
-    # Send coverage data to codecov.io
-    - bash .circle/codecov.sh
+    # Place reports in the appropriate folder
+    -  cp ${WORKDIR}/tests/*.xml ${CIRCLE_TEST_REPORTS}/tests/
 
 general:
   artifacts:
     - "~/work/docs"
     - "~/work/logs"
+    - "~/work/tests"
+    - "~/work/crashfiles"
 
 deployment:
   production:

Dockerfile

@@ -22,3 +22,7 @@ coverage:
         threshold: 100
         flags:
           - "smoketests"
+  ignore:          # files and folders that will be removed during processing
+    - "nipype/external/*"
+    - "tools/*"
+    - "doc/*"

circle.yml

@@ -8,7 +8,7 @@ arr=$@
 tmp_var=$( IFS=$' '; echo "${arr[*]}" )
 example_id=${tmp_var//[^A-Za-z0-9_-]/_}
 
-mkdir -p ${HOME}/.nipype ${WORKDIR}/logs/example_${example_id}
+mkdir -p ${HOME}/.nipype ${WORKDIR}/logs/example_${example_id} ${WORKDIR}/tests ${WORKDIR}/crashfiles
 echo "[logging]" > ${HOME}/.nipype/nipype.cfg
 echo "workflow_level = DEBUG" >> ${HOME}/.nipype/nipype.cfg
 echo "interface_level = DEBUG" >> ${HOME}/.nipype/nipype.cfg
@@ -17,23 +17,21 @@ echo "log_to_file = true" >> ${HOME}/.nipype/nipype.cfg
 echo "log_directory = ${WORKDIR}/logs/example_${example_id}" >> ${HOME}/.nipype/nipype.cfg
 
 # Set up coverage
-echo "[run]" >> .coveragerc
-echo "branch = True" >> .coveragerc
-echo "source = /src/nipype" >> .coveragerc
-echo "include = */nipype/*" >> .coveragerc
-echo "omit =" >> .coveragerc
-echo "    */nipype/external/*" >> .coveragerc
-echo "    */nipype/fixes/*" >> .coveragerc
-echo "    */setup.py" >> .coveragerc
+export COVERAGE_FILE=${WORKDIR}/tests/.coverage.${example_id}
+if [ "$2" == "MultiProc" ]; then
+	echo "concurrency = multiprocessing" >> /src/nipype/.coveragerc
+fi
 
+coverage run /src/nipype/tools/run_examples.py $@
+exit_code=$?
 
-parallel=""
-if [ "$2" == "MultiProc" ]; then
-	parallel="--concurrency=multiprocessing"
+# Generate coverage report and submit if token is defined
+coverage xml -o ${WORKDIR}/tests/smoketest_${example_id}.xml
+if [ "${CODECOV_TOKEN}" != "" ]; then
+	codecov.io -f ${WORKDIR}/tests/smoketest_${example_id}.xml -t "${CODECOV_TOKEN}" -F smoketests
 fi
 
-coverage run ${parallel} /src/nipype/tools/run_examples.py $@
-test_exit=$?
-coverage xml -o "${WORKDIR}/smoketest_${example_id}.xml"
+# Collect crashfiles
+find /work -name "crash-*" -maxdepth 1 -exec mv {} ${WORKDIR}/crashfiles/ \;
+exit $exit_code
 
-exit $test_exit

codecov.yml

@@ -3,11 +3,13 @@ set -e
 set -x
 set -u
 
+
+TESTPATH=${1:-/src/nipype/}
 WORKDIR=${WORK:-/work}
 PYTHON_VERSION=$( python -c "import sys; print('{}{}'.format(sys.version_info[0], sys.version_info[1]))" )
 
 # Create necessary directories
-mkdir -p ${WORKDIR}/crashfiles ${WORKDIR}/logs/py${PYTHON_VERSION}
+mkdir -p ${WORKDIR}/tests ${WORKDIR}/crashfiles ${WORKDIR}/logs/py${PYTHON_VERSION}
 
 # Create a nipype config file
 mkdir -p ${HOME}/.nipype
@@ -21,19 +23,28 @@ if [[ "${PYTHON_VERSION}" -lt "30" ]]; then
     echo 'profile_runtime = true' >> ${HOME}/.nipype/nipype.cfg
 fi
 
-cd /src/nipype/
-make clean
 # Run tests using pytest
-py.test -n ${CIRCLE_NCPUS:-4} --doctest-modules --junitxml=${WORKDIR}/pytests_py${PYTHON_VERSION}.xml --cov-report xml:${WORKDIR}/coverage_py${PYTHON_VERSION}.xml --cov=nipype nipype
+export COVERAGE_FILE=${WORKDIR}/tests/.coverage.py${PYTHON_VERSION}
+py.test -v --junitxml=${WORKDIR}/tests/pytests_py${PYTHON_VERSION}.xml --cov nipype --cov-config /src/nipype/.coveragerc --cov-report xml:${WORKDIR}/tests/coverage_py${PYTHON_VERSION}.xml ${TESTPATH}
+exit_code=$?
 
+if [ "${CODECOV_TOKEN}" != "" ]; then
+	codecov.io -f ${WORKDIR}/tests/coverage_py${PYTHON_VERSION}.xml -t "${CODECOV_TOKEN}" -F unittests
+fi
 
 # Workaround: run here the profiler tests in python 3
 if [[ "${PYTHON_VERSION}" -ge "30" ]]; then
     echo '[execution]' >> ${HOME}/.nipype/nipype.cfg
     echo 'profile_runtime = true' >> ${HOME}/.nipype/nipype.cfg
-    py.test -n ${CIRCLE_NCPUS:-4} --junitxml=${WORKDIR}/pytests_py${PYTHON_VERSION}_profiler.xml --cov-report xml:${WORKDIR}/coverage_py${PYTHON_VERSION}_profiler.xml --cov=nipype nipype/interfaces/tests/test_runtime_profiler.py
-    py.test -n ${CIRCLE_NCPUS:-4} --junitxml=${WORKDIR}/pytests_py${PYTHON_VERSION}_multiproc.xml --cov-report xml:${WORKDIR}/coverage_py${PYTHON_VERSION}_multiproc.xml --cov=nipype nipype/pipeline/plugins/tests/test_multiproc*.py
+    export COVERAGE_FILE=${WORKDIR}/tests/.coverage.py${PYTHON_VERSION}_extra
+    py.test -v --junitxml=${WORKDIR}/tests/pytests_py${PYTHON_VERSION}_extra.xml --cov nipype --cov-report xml:${WORKDIR}/tests/coverage_py${PYTHON_VERSION}_extra.xml /src/nipype/nipype/interfaces/tests/test_runtime_profiler.py /src/nipype/nipype/pipeline/plugins/tests/test_multiproc*.py
+    exit_code=$(( $exit_code + $? ))
+
+	if [ "${CODECOV_TOKEN}" != "" ]; then
+		codecov.io -f ${WORKDIR}/tests/coverage_py${PYTHON_VERSION}_extra.xml -t "${CODECOV_TOKEN}" -F unittests
+	fi
 fi
 
-# Copy crashfiles to scratch
-find /src/nipype/ -name "crash-*" -exec cp {} ${WORKDIR}/crashfiles/ \;
+# Collect crashfiles
+find /work -name "crash-*" -maxdepth 1 -exec mv {} ${WORKDIR}/crashfiles/ \;
+exit ${exit_code}