[ENH] Sorting out Dockerfiles

.dockerignore

@@ -1,7 +1,37 @@
-.git/
-*.pyc
-*.egg-info
+# python cache
+__pycache__/**/*
 __pycache__
-docker/nipype_*
-docker/test-*
-.coverage
+*.pyc
+
+# python distribution
+build/**/*
+build
+dist/**/*
+dist
+nipype.egg-info/**/*
+nipype.egg-info
+.eggs/**/*
+.eggs
+src/**/*
+src/
+
+# releasing
+Makefile
+
+# git
+.gitignore
+.git/**/*
+.git
+
+# other
+docs/**/*
+docs/
+.coverage
+.coveragerc
+codecov.yml
+rtd_requirements.txt
+circle.yml
+Vagrantfile
+.travis.yml
+.noserc
+

.gitignore

@@ -13,6 +13,7 @@
 .project
 .settings
 .pydevproject
+.eggs
 .idea/
 /documentation.zip
 .DS_Store
@@ -24,4 +25,4 @@ htmlcov/
 __pycache__/
 *~
 .ipynb_checkpoints/
-.ruby-version
+.ruby-version

CHANGES

@@ -1,6 +1,8 @@
 Upcoming release 0.13
 =====================
 
+* ENH: Revised all Dockerfiles and automated deployment to Docker Hub
+       from CircleCI (https://github.com/nipy/nipype/pull/1815)
 * FIX: Semaphore capture using MultiProc plugin (https://github.com/nipy/nipype/pull/1689)
 * REF: Refactor AFNI interfaces (https://github.com/nipy/nipype/pull/1678, https://github.com/nipy/nipype/pull/1680)
 * ENH: Move nipype commands to group command using click (https://github.com/nipy/nipype/pull/1608)

Dockerfile

@@ -26,49 +26,195 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-FROM nipype/testnipypedata:latest
-MAINTAINER Stanford Center for Reproducible Neuroscience <crn.poldracklab@gmail.com>
 
-# Preparations
-RUN ln -snf /bin/bash /bin/sh
+#
+# Based on https://github.com/poldracklab/fmriprep/blob/9c92a3de9112f8ef1655b876de060a2ad336ffb0/Dockerfile
+#
+FROM ubuntu:xenial-20161213
+
+# Prepare environment
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends curl bzip2 ca-certificates xvfb && \
+    curl -sSL http://neuro.debian.net/lists/xenial.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \
+    apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \
+    apt-get update
+
+# Installing freesurfer
+RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \
+    --exclude='freesurfer/trctrain' \
+    --exclude='freesurfer/subjects/fsaverage_sym' \
+    --exclude='freesurfer/subjects/fsaverage3' \
+    --exclude='freesurfer/subjects/fsaverage4' \
+    --exclude='freesurfer/subjects/fsaverage5' \
+    --exclude='freesurfer/subjects/fsaverage6' \
+    --exclude='freesurfer/subjects/cvs_avg35' \
+    --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \
+    --exclude='freesurfer/subjects/bert' \
+    --exclude='freesurfer/subjects/V1_average' \
+    --exclude='freesurfer/average/mult-comp-cor' \
+    --exclude='freesurfer/lib/cuda' \
+    --exclude='freesurfer/lib/qt'
+
+ENV FSL_DIR=/usr/share/fsl/5.0 \
+    OS=Linux \
+    FS_OVERRIDE=0 \
+    FIX_VERTEX_AREA= \
+    FSF_OUTPUT_FORMAT=nii.gz \
+    FREESURFER_HOME=/opt/freesurfer
+ENV SUBJECTS_DIR=$FREESURFER_HOME/subjects \
+    FUNCTIONALS_DIR=$FREESURFER_HOME/sessions \
+    MNI_DIR=$FREESURFER_HOME/mni \
+    LOCAL_DIR=$FREESURFER_HOME/local \
+    FSFAST_HOME=$FREESURFER_HOME/fsfast \
+    MINC_BIN_DIR=$FREESURFER_HOME/mni/bin \
+    MINC_LIB_DIR=$FREESURFER_HOME/mni/lib \
+    MNI_DATAPATH=$FREESURFER_HOME/mni/data \
+    FMRI_ANALYSIS_DIR=$FREESURFER_HOME/fsfast
+ENV PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \
+    MNI_PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \
+    PATH=$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH
+RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh
+
+# Installing Neurodebian packages (FSL, AFNI, git)
+RUN apt-get install -y --no-install-recommends \
+                    fsl-core=5.0.9-1~nd+1+nd16.04+1 \
+                    afni=16.2.07~dfsg.1-2~nd16.04+1
+
+ENV FSLDIR=/usr/share/fsl/5.0 \
+    FSLOUTPUTTYPE=NIFTI_GZ \
+    FSLMULTIFILEQUIT=TRUE \
+    POSSUMDIR=/usr/share/fsl/5.0 \
+    LD_LIBRARY_PATH=/usr/lib/fsl/5.0:$LD_LIBRARY_PATH \
+    FSLTCLSH=/usr/bin/tclsh \
+    FSLWISH=/usr/bin/wish \
+    AFNI_MODELPATH=/usr/lib/afni/models \
+    AFNI_IMSAVE_WARNINGS=NO \
+    AFNI_TTATLAS_DATASET=/usr/share/afni/atlases \
+    AFNI_PLUGINPATH=/usr/lib/afni/plugins \
+    PATH=/usr/lib/fsl/5.0:/usr/lib/afni/bin:$PATH
+
+# Installing and setting up ANTs
+RUN mkdir -p /opt/ants && \
+    curl -sSL "https://github.com/stnava/ANTs/releases/download/v2.1.0/Linux_Ubuntu14.04.tar.bz2" \
+    | tar -xjC /opt/ants --strip-components 1
+
+ENV ANTSPATH=/opt/ants \
+    PATH=$ANTSPATH:$PATH
+
+# Installing and setting up c3d
+RUN mkdir -p /opt/c3d && \
+    curl -sSL "http://downloads.sourceforge.net/project/c3d/c3d/1.0.0/c3d-1.0.0-Linux-x86_64.tar.gz" \
+    | tar -xzC /opt/c3d --strip-components 1
+
+ENV C3DPATH=/opt/c3d/ \
+    PATH=$C3DPATH/bin:$PATH
+
+# Install some other required tools
+RUN apt-get install -y --no-install-recommends \
+                    git=1:2.7.4-0ubuntu1 \
+                    graphviz=2.38.0-12ubuntu2 \
+                    unzip \
+                    apt-utils \
+                    fusefat \
+                    make \
+                    ruby=1:2.3.0+1 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install fake-S3
+ENV GEM_HOME /usr/lib/ruby/gems/2.3
+ENV BUNDLE_PATH="$GEM_HOME" \
+    BUNDLE_BIN="$GEM_HOME/bin" \
+    BUNDLE_SILENCE_ROOT_WARNING=1 \
+    BUNDLE_APP_CONFIG="$GEM_HOME"
+ENV PATH $BUNDLE_BIN:$PATH
+RUN mkdir -p "$GEM_HOME" "$BUNDLE_BIN" && \
+    chmod 777 "$GEM_HOME" "$BUNDLE_BIN"
+
+RUN gem install fakes3
+
+# Install Matlab MCR: from the good old install_spm_mcr.sh of @chrisfilo
+WORKDIR /opt
+RUN echo "destinationFolder=/opt/mcr" > mcr_options.txt && \
+    echo "agreeToLicense=yes" >> mcr_options.txt && \
+    echo "outputFile=/tmp/matlabinstall_log" >> mcr_options.txt && \
+    echo "mode=silent" >> mcr_options.txt && \
+    mkdir -p matlab_installer && \
+    curl -sSL http://www.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015a/installers/glnxa64/MCR_R2015a_glnxa64_installer.zip \
+         -o matlab_installer/installer.zip && \
+    unzip matlab_installer/installer.zip -d matlab_installer/ && \
+    matlab_installer/install -inputFile mcr_options.txt && \
+    rm -rf matlab_installer mcr_options.txt
+
+# Install SPM
+RUN curl -sSL http://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/dev/spm12_r6472_Linux_R2015a.zip -o spm12.zip && \
+    unzip spm12.zip && \
+    rm -rf spm12.zip
+
+ENV MATLABCMD="/opt/mcr/v85/toolbox/matlab" \
+    SPMMCRCMD="/opt/spm12/run_spm12.sh /opt/mcr/v85/ script" \
+    FORCE_SPMMCR=1
 
-# Install this branch's code
-WORKDIR /root/src
 
-# Install matplotlib, sphinx and coverage to build documentation
-# and run tests with coverage
-RUN source activate nipypetests-2.7 && \
-    pip install matplotlib sphinx coverage && \
-    source activate nipypetests-3.4 && \
-    pip install matplotlib sphinx coverage && \
-    source activate nipypetests-3.5 && \
-    pip install matplotlib sphinx coverage
+# Installing and setting up miniconda
+RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh && \
+    bash Miniconda3-4.2.12-Linux-x86_64.sh -b -p /usr/local/miniconda && \
+    rm Miniconda3-4.2.12-Linux-x86_64.sh
 
-ADD . nipype/
+ENV PATH=/usr/local/miniconda/bin:$PATH \
+    LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8 \
+    ACCEPT_INTEL_PYTHON_EULA=yes
 
-# Install the checked out version of nipype, check that requirements are
-# installed and install it for each of the three environments.
-RUN cd nipype/ && \
-    source activate nipypetests-2.7 && \
-    pip install -r requirements.txt && \
-    pip install -e .
+# Installing precomputed python packages
+RUN conda config --add channels conda-forge --add channels intel && \
+    chmod +x /usr/local/miniconda/bin/* && \
+    conda config --set always_yes yes --set changeps1 no && \
+    conda update -q conda && \
+    chmod +x /usr/local/miniconda/bin/* && \
+    conda install -y mkl=2017.0.1 \
+                     numpy=1.11.2 \
+                     scipy=0.18.1 \
+                     scikit-learn=0.17.1 \
+                     matplotlib=1.5.3 \
+                     pandas=0.19.0 \
+                     libxml2=2.9.4 \
+                     libxslt=1.1.29 \
+                     traits=4.6.0 \
+                     psutil=5.0.1 \
+                     icu=58.1
 
-RUN cd nipype/ && \
-    source activate nipypetests-3.4 && \
-    pip install -r requirements.txt && \
-    pip install -e .
+# matplotlib cleanups: set default backend, precaching fonts
+RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python3.5/site-packages/matplotlib/mpl-data/matplotlibrc && \
+    python -c "from matplotlib import font_manager"
 
-RUN cd nipype/ && \
-    source activate nipypetests-3.5 && \
-    pip install -r requirements.txt && \
-    pip install -e .
+# Unless otherwise specified each process should only use one thread - nipype
+# will handle parallelization
+ENV MKL_NUM_THREADS=1 \
+    OMP_NUM_THREADS=1
 
-WORKDIR /scratch
+# Installing dev requirements (packages that are not in pypi)
+WORKDIR /root/
+ADD requirements.txt requirements.txt
+RUN pip install -r requirements.txt && \
+    rm -rf ~/.cache/pip
 
-# Install entrypoints
-ADD docker/circleci/run_* /usr/bin/
-RUN chmod +x /usr/bin/run_*
+# Installing nipype
+COPY . /root/src/nipype
+RUN cd /root/src/nipype && \
+    pip install -e .[all] && \
+    rm -rf ~/.cache/pip
 
-# RUN echo 'source /etc/profile.d/nipype_tests.sh' >> /etc/bash.bashrc
-ENTRYPOINT ["/usr/bin/run_examples.sh"]
+WORKDIR /root/
 
+ARG BUILD_DATE
+ARG VCS_REF
+ARG VERSION
+LABEL org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.name="NIPYPE" \
+      org.label-schema.description="NIPYPE - Neuroimaging in Python: Pipelines and Interfaces" \
+      org.label-schema.url="http://nipype.readthedocs.io" \
+      org.label-schema.vcs-ref=$VCS_REF \
+      org.label-schema.vcs-url="https://github.com/nipy/nipype" \
+      org.label-schema.version=$VERSION \
+      org.label-schema.schema-version="1.0"

circle.yml

@@ -1,10 +1,13 @@
 machine:
+  pre:
+    - sudo curl -L -o /usr/bin/docker 'https://s3-external-1.amazonaws.com/circle-downloads/docker-1.9.1-circleci'
+    - sudo chmod 0755 /usr/bin/docker
   environment:
     OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage"
     DATA_NIPYPE_TUTORIAL_URL: "${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21"
     DATA_NIPYPE_FSL_COURSE: "${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe"
     DATA_NIPYPE_FSL_FEEDS: "${OSF_NIPYPE_URL}/57f473066c613b01f113e7af"
-
+    SCRATCH: "$HOME/scratch"
   services:
     - docker
 
@@ -18,31 +21,46 @@ dependencies:
     # Let CircleCI cache the apt archive
     - mkdir -p ~/.apt-cache/partial && sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives
     - sudo apt-get -y update && sudo apt-get install -y wget bzip2
-
+    # Create scratch folder and force group permissions
+    - mkdir -p $SCRATCH && sudo setfacl -d -m group:ubuntu:rwx $SCRATCH && sudo setfacl -m group:ubuntu:rwx $SCRATCH
+    - mkdir -p $HOME/docker $HOME/examples $SCRATCH/pytest $SCRATCH/logs
   override:
-    - mkdir -p ~/docker ~/examples ~/scratch/pytest ~/scratch/logs
     - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi
     - if [[ ! -d ~/examples/nipype-fsl_course_data ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-fsl_course_data.tar.gz "${DATA_NIPYPE_FSL_COURSE}" && tar xzf nipype-fsl_course_data.tar.gz -C ~/examples/; fi
     - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi
-    - if [[ -e ~/docker/image.tar ]]; then mv -n ~/docker/image.tar ~/docker/image_27.tar; fi
-    - if [[ -e ~/docker/image_27.tar ]]; then docker load -i ~/docker/image_27.tar; fi
-    - if [[ -e ~/docker/image_35.tar ]]; then docker load -i ~/docker/image_35.tar; fi
-    - docker build -f docker/nipype_test/Dockerfile_py27 -t nipype/nipype_test:py27 . :
+    - if [[ -e $HOME/docker/image.tar ]]; then docker load -i $HOME/docker/image.tar; fi
+    - if [[ -e $HOME/docker/image27.tar ]]; then docker load -i $HOME/docker/image27.tar; fi
+    - if [[ -e $HOME/docker/image35.tar ]]; then docker load -i $HOME/docker/image35.tar; fi
+    - sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
+    - e=1 && for i in {1..5}; do docker build -t nipype/nipype:latest --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
+        timeout: 21600
+    - docker build -f docker/Dockerfile_py27 -t nipype/nipype_test:py27 . :
         timeout: 1600
-    - docker build -f docker/nipype_test/Dockerfile_py35 -t nipype/nipype_test:py35 . :
-        timeout: 1600
-    - docker save nipype/nipype_test:py27 > ~/docker/image_27.tar :
-        timeout: 1600
-    - docker save nipype/nipype_test:py35 > ~/docker/image_35.tar :
+    - docker build -f docker/Dockerfile_py35 -t nipype/nipype_test:py35 . :
         timeout: 1600
+    - docker save nipype/nipype:latest > $HOME/docker/image.tar
+    - docker save nipype/nipype_test:py27 > $HOME/docker/image27.tar
+    - docker save nipype/nipype_test:py35 > $HOME/docker/image35.tar
 
 test:
   override:
-    - bash docker/circleci/tests.sh :
+    - bash docker/files/tests.sh :
         timeout: 7200
         parallel: true
 
 general:
   artifacts:
     - "~/docs"
     - "~/logs"
+
+deployment:
+  production:
+    tag: /.*/
+    commands:
+      - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push nipype/nipype:latest; fi :
+          timeout: 21600
+      - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag nipype/nipype nipype/nipype:$CIRCLE_TAG && docker push nipype/nipype:$CIRCLE_TAG; fi :
+          timeout: 21600
+# Automatic deployment to Pypi:
+#       - printf "[distutils]\nindex-servers =\n    pypi\n\n[pypi]\nusername:$PYPI_USER\npassword:$PYPI_PASS\n" > ~/.pypirc
+#       - python setup.py sdist upload -r pypi