Rearchitecture

.travis.yml

@@ -42,22 +42,22 @@ jobs:
           before_install:
             - ./continuous_integration/docker/hadoop/start.sh
           install:
-            - docker exec master /working/continuous_integration/docker/hadoop/install.sh
+            - ./continuous_integration/docker/hadoop/install.sh
           script:
-            - docker exec master /working/continuous_integration/docker/hadoop/script.sh
+            - ./continuous_integration/docker/hadoop/script.sh
 
-        - if: (commit_message !~ skip-tests) AND ((commit_message ~= test-all) OR (commit_message ~= test-kube))
-          env:
-            - KUBERNETES-TESTS=true
-          services:
-            - docker
-          before_install:
-            - source continuous_integration/before_install.sh
-            - ./continuous_integration/kubernetes/start.sh
-          install:
-            - ./continuous_integration/kubernetes/install.sh
-          script:
-            - ./continuous_integration/kubernetes/script.sh
+        #- if: (commit_message !~ skip-tests) AND ((commit_message ~= test-all) OR (commit_message ~= test-kube))
+          #env:
+            #- KUBERNETES-TESTS=true
+          #services:
+            #- docker
+          #before_install:
+            #- source continuous_integration/before_install.sh
+            #- ./continuous_integration/kubernetes/start.sh
+          #install:
+            #- ./continuous_integration/kubernetes/install.sh
+          #script:
+            #- ./continuous_integration/kubernetes/script.sh
 
         - if: (commit_message !~ skip-tests) AND ((commit_message ~= test-all) OR (commit_message ~= test-jobqueue) OR (commit_message ~= test-pbs))
           env:
@@ -67,9 +67,9 @@ jobs:
           before_install:
             - ./continuous_integration/docker/pbs/start.sh
           install:
-            - docker exec pbs /working/continuous_integration/docker/pbs/install.sh
+            - ./continuous_integration/docker/pbs/install.sh
           script:
-            - docker exec -u dask pbs /working/continuous_integration/docker/pbs/script.sh
+            - ./continuous_integration/docker/pbs/script.sh
 
         - if: (commit_message !~ skip-tests) AND ((commit_message ~= test-all) OR (commit_message ~= test-jobqueue) OR (commit_message ~= test-slurm))
           env:
@@ -79,26 +79,26 @@ jobs:
           before_install:
             - ./continuous_integration/docker/slurm/start.sh
           install:
-            - docker exec slurm /working/continuous_integration/docker/slurm/install.sh
+            - ./continuous_integration/docker/slurm/install.sh
           script:
-            - docker exec -u dask slurm /working/continuous_integration/docker/slurm/script.sh
+            - ./continuous_integration/docker/slurm/script.sh
 
-        - env:
-            - DOCS=true
-          before_install:
-            - source continuous_integration/before_install.sh
-          install:
-            - source continuous_integration/install.sh
-            - pip install kubernetes skein sphinx doctr dask-sphinx-theme
-          script:
-            - |
-              set -xe
-              pushd docs
-              make html
-              popd
-              if [[ "$TRAVIS_BRANCH" == "master" && "$TRAVIS_EVENT_TYPE" == "push" ]]; then
-                  doctr deploy . --built-docs docs/build/html/
-              fi
+        #- env:
+            #- DOCS=true
+          #before_install:
+            #- source continuous_integration/before_install.sh
+          #install:
+            #- source continuous_integration/install.sh
+            #- pip install kubernetes skein sphinx doctr dask-sphinx-theme
+          #script:
+            #- |
+              #set -xe
+              #pushd docs
+              #make html
+              #popd
+              #if [[ "$TRAVIS_BRANCH" == "master" && "$TRAVIS_EVENT_TYPE" == "push" ]]; then
+                  #doctr deploy . --built-docs docs/build/html/
+              #fi
 
 notifications:
   email: false

continuous_integration/docker/hadoop/_install.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+source ~/.bashrc
+
+set -xe
+
+cd /working
+
+conda install psutil pykerberos
+
+pip install \
+    aiohttp \
+    colorlog \
+    dask \
+    distributed \
+    cryptography \
+    traitlets \
+    sqlalchemy \
+    skein \
+    pytest \
+    pytest-asyncio
+
+pushd dask-gateway
+python setup.py develop
+popd
+
+pushd dask-gateway-server
+python setup.py develop
+popd
+
+pip list

continuous_integration/docker/hadoop/_script.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+source ~/.bashrc
+
+set -xe
+
+cd /working
+
+py.test tests/test_yarn_backend.py tests/test_auth.py -v

continuous_integration/docker/hadoop/install.sh

@@ -1,29 +1,2 @@
 #!/usr/bin/env bash
-source ~/.bashrc
-
-set -xe
-
-cd /working
-
-conda install psutil pykerberos
-
-pip install \
-    dask \
-    distributed \
-    cryptography \
-    tornado \
-    traitlets \
-    sqlalchemy \
-    skein \
-    pytest \
-    pytest-asyncio
-
-pushd dask-gateway
-python setup.py develop
-popd
-
-pushd dask-gateway-server
-python setup.py develop
-popd
-
-pip list
+docker exec master /working/continuous_integration/docker/hadoop/_install.sh

continuous_integration/docker/hadoop/script.sh

@@ -1,8 +1,2 @@
 #!/usr/bin/env bash
-source ~/.bashrc
-
-set -xe
-
-cd /working
-
-py.test tests/test_yarn_cluster.py tests/test_auth.py -v
+docker exec master /working/continuous_integration/docker/hadoop/_script.sh

continuous_integration/docker/hadoop/start.sh

@@ -9,5 +9,7 @@ docker run --rm -d \
     --name master \
     -h master.example.com \
     -v "$git_root":/working \
+    -p 8000:8000 \
+    -p 8786:8786 \
     -p 8088:8088 \
     daskgateway/testing-hadoop

continuous_integration/docker/pbs/_install.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+source ~/.bashrc
+
+set -xe
+
+cd /working
+
+conda install psutil
+
+pip install \
+    aiohttp \
+    colorlog \
+    dask \
+    distributed \
+    cryptography \
+    traitlets \
+    sqlalchemy \
+    pytest \
+    pytest-asyncio
+
+pushd dask-gateway
+python setup.py develop
+popd
+
+pushd dask-gateway-server
+python setup.py develop
+popd
+
+pip list

continuous_integration/docker/pbs/_script.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+source ~/.bashrc
+
+set -xe
+
+cd /working
+
+py.test tests/test_pbs_backend.py -v

continuous_integration/docker/pbs/install.sh

@@ -1,28 +1,2 @@
 #!/usr/bin/env bash
-source ~/.bashrc
-
-set -xe
-
-cd /working
-
-conda install psutil
-
-pip install \
-    dask \
-    distributed \
-    cryptography \
-    tornado \
-    traitlets \
-    sqlalchemy \
-    pytest \
-    pytest-asyncio
-
-pushd dask-gateway
-python setup.py develop
-popd
-
-pushd dask-gateway-server
-python setup.py develop
-popd
-
-pip list
+docker exec pbs /working/continuous_integration/docker/pbs/_install.sh

continuous_integration/docker/pbs/script.sh

@@ -1,8 +1,2 @@
 #!/usr/bin/env bash
-source ~/.bashrc
-
-set -xe
-
-cd /working
-
-py.test tests/test_pbs_cluster.py -v
+docker exec -u dask pbs /working/continuous_integration/docker/pbs/_script.sh

continuous_integration/docker/pbs/start.sh

@@ -9,6 +9,8 @@ docker run --rm -d \
     --name pbs \
     -h pbs \
     -v "$git_root":/working \
+    -p 8000:8000 \
+    -p 8786:8786 \
     -p 8088:8088 \
     --cap-add=SYS_RESOURCE \
     daskgateway/testing-pbs

continuous_integration/docker/slurm/_install.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+source ~/.bashrc
+
+set -xe
+
+cd /working
+
+conda install psutil
+
+pip install \
+    aiohttp \
+    colorlog \
+    dask \
+    distributed \
+    cryptography \
+    traitlets \
+    sqlalchemy \
+    pytest \
+    pytest-asyncio
+
+pushd dask-gateway
+python setup.py develop
+popd
+
+pushd dask-gateway-server
+python setup.py develop
+popd
+
+pip list

continuous_integration/docker/slurm/_script.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+source ~/.bashrc
+
+set -xe
+
+cd /working
+
+py.test tests/test_slurm_backend.py -v

continuous_integration/docker/slurm/install.sh

@@ -1,28 +1,2 @@
 #!/usr/bin/env bash
-source ~/.bashrc
-
-set -xe
-
-cd /working
-
-conda install psutil
-
-pip install \
-    dask \
-    distributed \
-    cryptography \
-    tornado \
-    traitlets \
-    sqlalchemy \
-    pytest \
-    pytest-asyncio
-
-pushd dask-gateway
-python setup.py develop
-popd
-
-pushd dask-gateway-server
-python setup.py develop
-popd
-
-pip list
+docker exec slurm /working/continuous_integration/docker/slurm/_install.sh

continuous_integration/docker/slurm/script.sh

@@ -1,8 +1,2 @@
 #!/usr/bin/env bash
-source ~/.bashrc
-
-set -xe
-
-cd /working
-
-py.test tests/test_slurm_cluster.py -v
+docker exec -u dask slurm /working/continuous_integration/docker/slurm/_script.sh

continuous_integration/docker/slurm/start.sh

@@ -9,4 +9,7 @@ docker run --rm -d \
     --name slurm \
     -h slurm \
     -v "$git_root":/working \
+    -p 8000:8000 \
+    -p 8786:8786 \
+    -p 8088:8088 \
     daskgateway/testing-slurm

continuous_integration/install.sh

@@ -3,20 +3,22 @@ set -xe
 npm install -g configurable-http-proxy
 
 pip install \
+    aiohttp \
+    black \
+    colorlog \
+    cryptography \
     dask \
     distributed \
-    cryptography \
-    tornado \
-    traitlets \
-    sqlalchemy \
-    pytest \
-    pytest-asyncio \
-    trustme \
-    jupyterhub \
+    flake8 \
     ipywidgets \
+    jupyterhub \
     notebook \
-    black \
-    flake8
+    pytest \
+    pytest-asyncio \
+    sqlalchemy \
+    tornado \
+    traitlets \
+    trustme
 
 pushd dask-gateway
 python setup.py develop