Update our master branch with runtime profiling and nipype re-base

.gitignore

@@ -5,6 +5,7 @@
 /nipype/build
 /nipype/nipype.egg-info
 /doc/_build
+/doc/preproc
 /doc/users/examples
 /doc/api/generated
 *.pyc
@@ -17,3 +18,6 @@
 .DS_Store
 nipype/testing/data/von-ray_errmap.nii.gz
 nipype/testing/data/von_errmap.nii.gz
+crash*.pklz
+.coverage
+htmlcov/

.noserc

@@ -0,0 +1,11 @@
+[nosetests]
+verbosity=3
+
+with-coverage=1
+cover-branches=1
+cover-xml=1
+cover-xml-file=./coverage.xml
+cover-min-percentage=50
+
+
+with-xunit=1

.travis.yml

@@ -3,16 +3,17 @@ cache:
 language: python
 python:
 - 2.7
+- 3.4
 env:
 - INSTALL_DEB_DEPENDECIES=true
 - INSTALL_DEB_DEPENDECIES=false
 before_install:
-- if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh
-  -O miniconda.sh; else wget http://repo.continuum.io/miniconda/Miniconda3-3.6.0-Linux-x86_64.sh
+- if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then wget http://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh
+  -O miniconda.sh; else wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
   -O miniconda.sh; fi
 - chmod +x miniconda.sh
 - "./miniconda.sh -b"
-- export PATH=/home/travis/miniconda/bin:$PATH
+- if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then export PATH=/home/travis/miniconda2/bin:$PATH; else export PATH=/home/travis/miniconda3/bin:$PATH; fi
 - if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi
 - if $INSTALL_DEB_DEPENDECIES; then sudo ln -s /run/shm /dev/shm; fi
 - if $INSTALL_DEB_DEPENDECIES; then bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh);
@@ -22,20 +23,21 @@ before_install:
 - if $INSTALL_DEB_DEPENDECIES; then travis_retry sudo apt-get install -qq fsl-atlases;
   fi
 - if $INSTALL_DEB_DEPENDECIES; then source /etc/fsl/fsl.sh; fi
+- if $INSTALL_DEB_DEPENDECIES; then source /etc/afni/afni.sh; fi
+- export FSLOUTPUTTYPE=NIFTI_GZ
 install:
 - conda update --yes conda
 - conda create -n testenv --yes pip python=$TRAVIS_PYTHON_VERSION
 - source activate testenv
 - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then pip install ordereddict; fi
-- conda install --yes numpy scipy nose traits networkx dateutil
-- pip install nibabel
+- conda install --yes numpy scipy nose networkx dateutil
+- if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then conda install --yes traits; else pip install traits; fi
 - pip install python-coveralls
 - pip install nose-cov
-- pip install https://github.com/RDFLib/rdflib/archive/master.zip
-- pip install https://github.com/trungdong/prov/archive/rdf.zip
+- pip install -r requirements.txt  # finish remaining requirements
 - python setup.py install
 script:
-- nosetests --with-doctest --with-cov --cov nipype --cov-config .coveragerc --logging-level=INFO
+- python -W once:FSL:UserWarning:nipype `which nosetests` --with-doctest --with-cov --cover-package nipype --cov-config .coveragerc --logging-level=INFO
 after_success:
 - coveralls --config_file .coveragerc
 deploy:

CHANGES

@@ -1,7 +1,20 @@
-Next Release
+Next release
 ============
+
+* ENH: Added support for PETPVC (https://github.com/nipy/nipype/pull/1335)
+* ENH: Merge S3DataSink into DataSink, added AWS documentation (https://github.com/nipy/nipype/pull/1316)
+* TST: Cache APT in CircleCI (https://github.com/nipy/nipype/pull/1333)
+* ENH: Add new flags to the BRAINSABC for new features (https://github.com/nipy/nipype/pull/1322)
+* ENH: Provides a Nipype wrapper for ANTs DenoiseImage (https://github.com/nipy/nipype/pull/1291)
+* FIX: Minor bugfix logging hash differences (https://github.com/nipy/nipype/pull/1298)
+* FIX: Use released Prov python library (https://github.com/nipy/nipype/pull/1279)
+* ENH: Support for Python 3 (https://github.com/nipy/nipype/pull/1221)
+* FIX: VTK version check missing when using tvtk (https://github.com/nipy/nipype/pull/1219)
+* ENH: Added an OAR scheduler plugin (https://github.com/nipy/nipype/pull/1259)
+* ENH: New ANTs interface: antsBrainExtraction (https://github.com/nipy/nipype/pull/1231)
 * API: Default model level for the bedpostx workflow has been set to "2" following FSL 5.0.9 lead
 * ENH: New interfaces for interacting with AWS S3: S3DataSink and S3DataGrabber (https://github.com/nipy/nipype/pull/1201)
+* ENH: Interfaces for MINC tools (https://github.com/nipy/nipype/pull/1304)
 
 Release 0.11.0 (September 15, 2015)
 ============

THANKS.rst

@@ -10,9 +10,11 @@ Contributors to Nipype include but are not limited to:
   * Aimi Watanabe
   * Alexander Schaefer
   * Alexandre Gramfort
+  * Alexandre Savio
   * Anisha Keshavan
   * Ariel Rokem
   * Ben Acland
+  * Ben Cipollini
   * Basile Pinsard
   * Brendan Moloney
   * Brian Cheung

bin/nipype2boutiques

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
 # vi: set ft=python sts=4 ts=4 sw=4 et:
 import sys

bin/nipype_cmd

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
 # vi: set ft=python sts=4 ts=4 sw=4 et:
 import sys

bin/nipype_display_crash

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!python
 """Displays crash information from Nipype crash files. For certain crash files,
 one can rerun a failed node in a temp directory.
 

build_docs.py

@@ -8,6 +8,8 @@
     python setup.py build_sphinx
 """
 
+from __future__ import print_function
+
 # Standard library imports
 import sys
 import os
@@ -27,23 +29,25 @@
 
 ################################################################################
 # Distutils Command class for installing nipype to a temporary location.
+
+
 class TempInstall(Command):
     temp_install_dir = os.path.join('build', 'install')
 
     def run(self):
         """ build and install nipype in a temporary location. """
         install = self.distribution.get_command_obj('install')
         install.install_scripts = self.temp_install_dir
-        install.install_base    = self.temp_install_dir
+        install.install_base = self.temp_install_dir
         install.install_platlib = self.temp_install_dir
         install.install_purelib = self.temp_install_dir
-        install.install_data    = self.temp_install_dir
-        install.install_lib     = self.temp_install_dir
+        install.install_data = self.temp_install_dir
+        install.install_lib = self.temp_install_dir
         install.install_headers = self.temp_install_dir
         install.run()
 
         # Horrible trick to reload nipype with our temporary instal
-        for key in sys.modules.keys():
+        for key in list(sys.modules.keys()):
             if key.startswith('nipype'):
                 sys.modules.pop(key, None)
         sys.path.append(os.path.abspath(self.temp_install_dir))
@@ -62,12 +66,11 @@ def finalize_options(self):
 # Distutils Command class for API generation
 class APIDocs(TempInstall):
     description = \
-    """generate API docs """
+        """generate API docs """
 
     user_options = [
         ('None', None, 'this command has no options'),
-        ]
-
+    ]
 
     def run(self):
         # First build the project and install it to a temporary location.
@@ -131,30 +134,29 @@ def run(self):
 
         def zip_docs(self):
             if not os.path.exists(DOC_BUILD_DIR):
-                raise OSError, 'Doc directory does not exist.'
+                raise OSError('Doc directory does not exist.')
             target_file = os.path.join('doc', 'documentation.zip')
             # ZIP_DEFLATED actually compresses the archive. However, there
             # will be a RuntimeError if zlib is not installed, so we check
             # for it. ZIP_STORED produces an uncompressed zip, but does not
             # require zlib.
             try:
                 zf = zipfile.ZipFile(target_file, 'w',
-                                            compression=zipfile.ZIP_DEFLATED)
+                                     compression=zipfile.ZIP_DEFLATED)
             except RuntimeError:
                 warnings.warn('zlib not installed, storing the docs '
-                                'without compression')
+                              'without compression')
                 zf = zipfile.ZipFile(target_file, 'w',
-                                            compression=zipfile.ZIP_STORED)
+                                     compression=zipfile.ZIP_STORED)
 
             for root, dirs, files in os.walk(DOC_BUILD_DIR):
                 relative = relative_path(root)
                 if not relative.startswith('.doctrees'):
                     for f in files:
                         zf.write(os.path.join(root, f),
-                                os.path.join(relative, 'html_docs', f))
+                                 os.path.join(relative, 'html_docs', f))
             zf.close()
 
-
         def finalize_options(self):
             """ Override the default for the documentation build
                 directory.
@@ -164,23 +166,25 @@ def finalize_options(self):
 
 ################################################################################
 # Distutils Command class to clean
+
+
 class Clean(clean):
 
     def run(self):
         clean.run(self)
         api_path = os.path.join('doc', 'api', 'generated')
         if os.path.exists(api_path):
-            print "Removing %s" % api_path
+            print("Removing %s" % api_path)
             shutil.rmtree(api_path)
         interface_path = os.path.join('doc', 'interfaces', 'generated')
         if os.path.exists(interface_path):
-            print "Removing %s" % interface_path
+            print("Removing %s" % interface_path)
             shutil.rmtree(interface_path)
         if os.path.exists(DOC_BUILD_DIR):
-            print "Removing %s" % DOC_BUILD_DIR
+            print("Removing %s" % DOC_BUILD_DIR)
             shutil.rmtree(DOC_BUILD_DIR)
         if os.path.exists(DOC_DOCTREES_DIR):
-            print "Removing %s" % DOC_DOCTREES_DIR
+            print("Removing %s" % DOC_DOCTREES_DIR)
             shutil.rmtree(DOC_DOCTREES_DIR)
 
 
@@ -189,5 +193,3 @@ def run(self):
             'api_docs': APIDocs,
             'clean': Clean,
             }
-
-

circle.yml

@@ -1,83 +1,71 @@
 dependencies:
   cache_directories:
+    - "~/.apt-cache"
     - "~/examples/data"
     - "~/examples/fsdata"
-    - "~/examples/feeds"
     - "~/mcr"
     - "~/spm12"
-    - "~/fsl"
     - "~/examples/fsl_course_data"
+  pre:
+    # Let CircleCI cache the apt archive
+    - sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial
+    - wget -O- http://neuro.debian.net/lists/precise.us-ca.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list
+    - sudo apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9
+    - sudo apt-get update
   override:
+    # Install apt packages
+    - sudo apt-get install -y fsl-core fsl-atlases fsl-mni152-templates fsl-feeds afni
+    - echo "source /etc/fsl/fsl.sh" >> $HOME/.profile
+    - echo "source /etc/afni/afni.sh" >> $HOME/.profile
+    - mkdir -p ~/examples/ && ln -sf /usr/share/fsl-feeds/ ~/examples/feeds
+    # Set up python environment
     - pip install --upgrade pip
     - pip install -e .
-    - pip install matplotlib sphinx ipython boto
+    - pip install matplotlib sphinx ipython boto coverage
     - gem install fakes3
-    - if [[ ! -d ~/fsl ]]; then wget "http://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-5.0.9-centos6_64.tar.gz"; tar zxvf fsl-5.0.9-centos6_64.tar.gz; mv fsl ~/fsl; fi
-    - if [[ ! -d ~/examples/data ]]; then wget "http://tcpdiag.dl.sourceforge.net/project/nipy/nipype/nipype-0.2/nipype-tutorial.tar.bz2"; tar jxvf nipype-tutorial.tar.bz2; mkdir ~/examples; mv nipype-tutorial/* ~/examples/; fi
-    # we download this manually because CircleCI does not cache apt
-    - if [[ ! -d ~/examples/feeds ]]; then wget "http://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-5.0.8-feeds.tar.gz"; tar zxvf fsl-5.0.8-feeds.tar.gz; mv feeds ~/examples/; fi
-    - if [[ ! -d ~/examples/fsl_course_data ]]; then wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" ; wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz"; wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz"; mkdir ~/examples/fsl_course_data; tar zxvf fdt1.tar.gz -C ~/examples/fsl_course_data; tar zxvf fdt2.tar.gz -C ~/examples/fsl_course_data; tar zxvf tbss.tar.gz -C ~/examples/fsl_course_data; fi
+    - if [[ ! -d ~/examples/data ]]; then wget "http://tcpdiag.dl.sourceforge.net/project/nipy/nipype/nipype-0.2/nipype-tutorial.tar.bz2" && tar jxvf nipype-tutorial.tar.bz2 && mv nipype-tutorial/* ~/examples/; fi
+    - if [[ ! -d ~/examples/fsl_course_data ]]; then wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz" && wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz" && mkdir ~/examples/fsl_course_data && tar zxvf fdt1.tar.gz -C ~/examples/fsl_course_data && tar zxvf fdt2.tar.gz -C ~/examples/fsl_course_data && tar zxvf tbss.tar.gz -C ~/examples/fsl_course_data; fi
     - bash ~/nipype/tools/install_spm_mcr.sh
-    - mkdir -p ~/.nipype && echo "[logging]" > ~/.nipype/nipype.cfg && echo "workflow_level = DEBUG" >> ~/.nipype/nipype.cfg && echo "interface_level = DEBUG" >> ~/.nipype/nipype.cfg && echo "filemanip_level = DEBUG" >> ~/.nipype/nipype.cfg
+    - mkdir -p ~/.nipype && echo '[logging]' > ~/.nipype/nipype.cfg && echo 'workflow_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'interface_level = DEBUG' >> ~/.nipype/nipype.cfg && echo 'filemanip_level = DEBUG' >> ~/.nipype/nipype.cfg
+machine:
+  environment:
+    FSLOUTPUTTYPE: NIFTI_GZ
 test:
   override:
-    - nosetests --with-doctest --logging-level=DEBUG --verbosity=3:
+    - mkdir -p ${CIRCLE_TEST_REPORTS}/nose
+    - source $HOME/.profile; nosetests --with-doctest --xunit-file="${CIRCLE_TEST_REPORTS}/nose/${CIRCLE_PROJECT_REPONAME}.xml" -c ./.noserc --logging-level=DEBUG --verbosity=3:
         environment:
           SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
           FORCE_SPMMCR: 1
           FSL_COURSE_DATA: "$HOME/examples/fsl_course_data"
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
         timeout: 2600
     - set -o pipefail && cd doc && make html 2>&1 | tee ~/log.txt
     - cat ~/log.txt && if grep -q "ERROR" ~/log.txt; then false; else true; fi
-    - python ~/nipype/tools/run_examples.py test_spm Linear workflow3d workflow4d:
+    - source $HOME/.profile; python ~/nipype/tools/run_examples.py test_spm Linear workflow3d workflow4d:
         pwd: ../examples
         environment:
           SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
           FORCE_SPMMCR: 1
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
         timeout: 1600
-    - python ~/nipype/tools/run_examples.py fmri_fsl_feeds Linear l1pipeline:
+    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_fsl_feeds Linear l1pipeline:
         pwd: ../examples
-        environment:
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
-    - python ~/nipype/tools/run_examples.py fmri_spm_dartel Linear level1 l2pipeline:
+    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_spm_dartel Linear level1 l2pipeline:
         pwd: ../examples
         environment:
           SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
           FORCE_SPMMCR: 1
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
         timeout: 1600
-    - python ~/nipype/tools/run_examples.py fmri_fsl_reuse Linear level1_workflow:
+    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_fsl_reuse Linear level1_workflow:
         pwd: ../examples
-        environment:
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
-    - python ~/nipype/tools/run_examples.py fmri_spm_nested Linear level1 l2pipeline:
+    - source $HOME/.profile; python ~/nipype/tools/run_examples.py fmri_spm_nested Linear level1 l2pipeline:
         pwd: ../examples
         environment:
           SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
           FORCE_SPMMCR: 1
-          FSLDIR: "$HOME/fsl/"
-          PATH: "$HOME/fsl/bin:$PATH"
-          LD_LIBRARY_PATH: "$HOME/fsl/lib"
-          FSLOUTPUTTYPE: "NIFTI_GZ"
 
 general:
   artifacts:
     - "doc/_build/html"
     - "~/log.txt"
+    - "nosetests.xml"
+    - "coverage.xml"